RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4311 from Gallaecio/metarefresh-ignore-tags 

Make METAREFRESH_IGNORE_TAGS an empty list by default
This commit is contained in:
Mikhail Korobov 2020-02-06 23:40:45 +05:00 committed by GitHub
parent 8d2705f23c 576663e5a7
commit 042e71e2b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 11 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/topics/downloader-middleware.rst
View File

@ -868,7 +868,7 @@ Whether the Meta Refresh middleware will be enabled.
METAREFRESH_IGNORE_TAGS
^^^^^^^^^^^^^^^^^^^^^^^
Default: ``['script', 'noscript']``
Default: ``[]``
Meta tags within these tags are ignored.

2
scrapy/settings/default_settings.py
View File

@ -223,7 +223,7 @@ MEMUSAGE_NOTIFY_MAIL = []
MEMUSAGE_WARNING_MB = 0
METAREFRESH_ENABLED = True
METAREFRESH_IGNORE_TAGS = ['script', 'noscript']
METAREFRESH_IGNORE_TAGS = []
METAREFRESH_MAXDELAY = 100
NEWSPIDER_MODULE = ''

16
tests/test_downloadermiddleware_redirect.py
View File

@ -300,19 +300,21 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
body = ('''<noscript><meta http-equiv="refresh" '''
'''content="0;URL='http://example.org/newpage'"></noscript>''')
rsp = HtmlResponse(req.url, body=body.encode())
response = self.mw.process_response(req, rsp, self.spider)
assert isinstance(response, Response)
req2 = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)
self.assertEqual(req2.url, 'http://example.org/newpage')
def test_ignore_tags_empty_list(self):
crawler = get_crawler(Spider, {'METAREFRESH_IGNORE_TAGS': []})
def test_ignore_tags_1_x_list(self):
"""Test that Scrapy 1.x behavior remains possible"""
settings = {'METAREFRESH_IGNORE_TAGS': ['script', 'noscript']}
crawler = get_crawler(Spider, settings)
mw = MetaRefreshMiddleware.from_crawler(crawler)
req = Request(url='http://example.org')
body = ('''<noscript><meta http-equiv="refresh" '''
'''content="0;URL='http://example.org/newpage'"></noscript>''')
rsp = HtmlResponse(req.url, body=body.encode())
req2 = mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)
self.assertEqual(req2.url, 'http://example.org/newpage')
response = mw.process_response(req, rsp, self.spider)
assert isinstance(response, Response)
if __name__ == "__main__":
unittest.main()