From 180bc9bad7aceb3a9e10c1411212914bc32fb721 Mon Sep 17 00:00:00 2001 From: aisha-partha <153170327+aisha-partha@users.noreply.github.com> Date: Wed, 8 May 2024 22:06:46 +0530 Subject: [PATCH] =?UTF-8?q?Closes=20#6342.=20Setting=20METAREFRESH=5FIGNOR?= =?UTF-8?q?E=5FTAGS=20to=20[=E2=80=98noscript=E2=80=99]=20by=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/downloader-middleware.rst | 4 ++-- scrapy/settings/default_settings.py | 2 +- tests/test_downloadermiddleware_redirect.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1abbc4968..3f90cf2ed 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -876,13 +876,13 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``[]`` +Default: ``['noscript']`` Meta tags within these tags are ignored. .. versionchanged:: 2.0 The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``['script', 'noscript']`` to ``[]``. + ``[]`` to ``['noscript']``. .. setting:: METAREFRESH_MAXDELAY diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2b3d95a0e..d7ac7ec35 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -239,7 +239,7 @@ MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_WARNING_MB = 0 METAREFRESH_ENABLED = True -METAREFRESH_IGNORE_TAGS = [] +METAREFRESH_IGNORE_TAGS = ["noscript"] METAREFRESH_MAXDELAY = 100 NEWSPIDER_MODULE = "" diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 10b8ca9af..83ff25982 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -395,9 +395,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): """content="0;URL='http://example.org/newpage'">""" ) rsp = HtmlResponse(req.url, body=body.encode()) - req2 = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, "http://example.org/newpage") + response = self.mw.process_response(req, rsp, self.spider) + assert isinstance(response, Response) def test_ignore_tags_1_x_list(self): """Test that Scrapy 1.x behavior remains possible"""