RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ignore meta-refresh redirect when embedded inside <noscript> tag. closes issue 18

This commit is contained in:
Pablo Hoffman 2011-11-14 16:25:22 -02:00
parent 6cc40dc062
commit ec1ef0235f
2 changed files with 21 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
scrapy/tests/test_utils_response.py
View File

@ -3,7 +3,8 @@ import unittest
import urlparse
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.response import body_or_str, response_httprepr, open_in_browser
from scrapy.utils.response import body_or_str, response_httprepr, open_in_browser, \
get_meta_refresh
__doctests__ = ['scrapy.utils.response']
@ -55,5 +56,21 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertRaises(TypeError, open_in_browser, Response(url, body=body), \
debug=True)
def test_get_meta_refresh(self):
r1 = HtmlResponse("http://www.example.com", body="""
<html>
<head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
<body>blahablsdfsal&amp;</body>
</html>""")
r2 = HtmlResponse("http://www.example.com", body="""
<html>
<head><title>Dummy</title><noScript>
<meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
</noSCRIPT>
<body>blahablsdfsal&amp;</body>
</html>""")
self.assertEqual(get_meta_refresh(r1), (5.0, 'http://example.org/newpage'))
self.assertEqual(get_meta_refresh(r2), (None, None))
if __name__ == "__main__":
unittest.main()

4
scrapy/utils/response.py
View File

@ -4,6 +4,7 @@ scrapy.http.Response objects
"""
import os
import re
import weakref
import webbrowser
import tempfile
@ -12,7 +13,6 @@ from twisted.web import http
from twisted.web.http import RESPONSES
from w3lib import html
from scrapy.xlib.BeautifulSoup import BeautifulSoup
from scrapy.http import Response, HtmlResponse
def body_or_str(obj, unicode=True):
@ -34,11 +34,13 @@ def get_base_url(response):
response.encoding)
return _baseurl_cache[response]
_noscript_re = re.compile(u'<noscript>.*?</noscript>', re.IGNORECASE | re.DOTALL)
_metaref_cache = weakref.WeakKeyDictionary()
def get_meta_refresh(response):
"""Parse the http-equiv refrsh parameter from the given response"""
if response not in _metaref_cache:
text = response.body_as_unicode()[0:4096]
text = _noscript_re.sub(u'', text)
_metaref_cache[response] = html.get_meta_refresh(text, response.url, \
response.encoding)
return _metaref_cache[response]