diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py
index 6cde6332c..9e75a7c7f 100644
--- a/scrapy/tests/test_utils_response.py
+++ b/scrapy/tests/test_utils_response.py
@@ -3,7 +3,8 @@ import unittest
import urlparse
from scrapy.http import Response, TextResponse, HtmlResponse
-from scrapy.utils.response import body_or_str, response_httprepr, open_in_browser
+from scrapy.utils.response import body_or_str, response_httprepr, open_in_browser, \
+ get_meta_refresh
__doctests__ = ['scrapy.utils.response']
@@ -55,5 +56,21 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertRaises(TypeError, open_in_browser, Response(url, body=body), \
debug=True)
+ def test_get_meta_refresh(self):
+ r1 = HtmlResponse("http://www.example.com", body="""
+
+ Dummy
+ blahablsdfsal&
+ """)
+ r2 = HtmlResponse("http://www.example.com", body="""
+
+ Dummy
+ blahablsdfsal&
+ """)
+ self.assertEqual(get_meta_refresh(r1), (5.0, 'http://example.org/newpage'))
+ self.assertEqual(get_meta_refresh(r2), (None, None))
+
if __name__ == "__main__":
unittest.main()
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 668e344f0..daa3a62bc 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -4,6 +4,7 @@ scrapy.http.Response objects
"""
import os
+import re
import weakref
import webbrowser
import tempfile
@@ -12,7 +13,6 @@ from twisted.web import http
from twisted.web.http import RESPONSES
from w3lib import html
-from scrapy.xlib.BeautifulSoup import BeautifulSoup
from scrapy.http import Response, HtmlResponse
def body_or_str(obj, unicode=True):
@@ -34,11 +34,13 @@ def get_base_url(response):
response.encoding)
return _baseurl_cache[response]
+_noscript_re = re.compile(u'', re.IGNORECASE | re.DOTALL)
_metaref_cache = weakref.WeakKeyDictionary()
def get_meta_refresh(response):
"""Parse the http-equiv refrsh parameter from the given response"""
if response not in _metaref_cache:
text = response.body_as_unicode()[0:4096]
+ text = _noscript_re.sub(u'', text)
_metaref_cache[response] = html.get_meta_refresh(text, response.url, \
response.encoding)
return _metaref_cache[response]