From ec1ef0235f9deee0c263c9b31652d3e74a754acc Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Mon, 14 Nov 2011 16:25:22 -0200
Subject: [PATCH] ignore meta-refresh redirect when embedded inside <noscript>
 tag. closes issue 18

---
 scrapy/tests/test_utils_response.py | 19 ++++++++++++++++++-
 scrapy/utils/response.py            |  4 +++-
 2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py
index 6cde6332c..9e75a7c7f 100644
--- a/scrapy/tests/test_utils_response.py
+++ b/scrapy/tests/test_utils_response.py
@@ -3,7 +3,8 @@ import unittest
 import urlparse
 
 from scrapy.http import Response, TextResponse, HtmlResponse
-from scrapy.utils.response import body_or_str, response_httprepr, open_in_browser
+from scrapy.utils.response import body_or_str, response_httprepr, open_in_browser, \
+    get_meta_refresh
 
 __doctests__ = ['scrapy.utils.response']
 
@@ -55,5 +56,21 @@ class ResponseUtilsTest(unittest.TestCase):
         self.assertRaises(TypeError, open_in_browser, Response(url, body=body), \
             debug=True)
 
+    def test_get_meta_refresh(self):
+        r1 = HtmlResponse("http://www.example.com", body="""
+        <html>
+        <head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
+        <body>blahablsdfsal&amp;</body>
+        </html>""")
+        r2 = HtmlResponse("http://www.example.com", body="""
+        <html>
+        <head><title>Dummy</title><noScript>
+        <meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
+        </noSCRIPT>
+        <body>blahablsdfsal&amp;</body>
+        </html>""")
+        self.assertEqual(get_meta_refresh(r1), (5.0, 'http://example.org/newpage'))
+        self.assertEqual(get_meta_refresh(r2), (None, None))
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 668e344f0..daa3a62bc 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -4,6 +4,7 @@ scrapy.http.Response objects
 """
 
 import os
+import re
 import weakref
 import webbrowser
 import tempfile
@@ -12,7 +13,6 @@ from twisted.web import http
 from twisted.web.http import RESPONSES
 from w3lib import html
 
-from scrapy.xlib.BeautifulSoup import BeautifulSoup
 from scrapy.http import Response, HtmlResponse
 
 def body_or_str(obj, unicode=True):
@@ -34,11 +34,13 @@ def get_base_url(response):
             response.encoding)
     return _baseurl_cache[response]
 
+_noscript_re = re.compile(u'<noscript>.*?</noscript>', re.IGNORECASE | re.DOTALL)
 _metaref_cache = weakref.WeakKeyDictionary()
 def get_meta_refresh(response):
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.body_as_unicode()[0:4096]
+        text = _noscript_re.sub(u'', text)
         _metaref_cache[response] = html.get_meta_refresh(text, response.url, \
             response.encoding)
     return _metaref_cache[response]
