RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Set METAREFRESH_IGNORE_TAGS to ['noscript'] by default (#6347)

This commit is contained in:
aisha-partha 2024-05-08 22:06:46 +05:30 committed by Adrián Chaves
parent 397d21f1f5
commit 3562618f67
3 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/topics/downloader-middleware.rst
View File

@ -884,6 +884,10 @@ Meta tags within these tags are ignored.
The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
``['script', 'noscript']`` to ``[]``.
.. versionchanged:: VERSION
The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
``[]`` to ``['noscript']``.
.. setting:: METAREFRESH_MAXDELAY
METAREFRESH_MAXDELAY

2
scrapy/settings/default_settings.py
View File

@ -237,7 +237,7 @@ MEMUSAGE_NOTIFY_MAIL = []
MEMUSAGE_WARNING_MB = 0
METAREFRESH_ENABLED = True
METAREFRESH_IGNORE_TAGS = []
METAREFRESH_IGNORE_TAGS = ["noscript"]
METAREFRESH_MAXDELAY = 100
NEWSPIDER_MODULE = ""

5
tests/test_downloadermiddleware_redirect.py
View File

@ -395,9 +395,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
"""content="0;URL='http://example.org/newpage'"></noscript>"""
)
rsp = HtmlResponse(req.url, body=body.encode())
req2 = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)
self.assertEqual(req2.url, "http://example.org/newpage")
response = self.mw.process_response(req, rsp, self.spider)
assert isinstance(response, Response)
def test_ignore_tags_1_x_list(self):
"""Test that Scrapy 1.x behavior remains possible"""