diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 9ade8472f..c14b3f13f 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,21 +1,13 @@ import asyncio import logging -import platform import re -import signal -import subprocess -import sys import warnings -from abc import ABC, abstractmethod from collections.abc import Generator from pathlib import Path from typing import Any import pytest -from packaging.version import parse as parse_version -from pexpect.popen_spawn import PopenSpawn from twisted.internet.defer import Deferred -from w3lib import __version__ as w3lib_version from zope.interface.exceptions import MultipleInvalid import scrapy @@ -30,7 +22,6 @@ from scrapy.crawler import ( from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings -from scrapy.utils.asyncio import call_later from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.log import ( _uninstall_scrapy_root_handler, @@ -39,8 +30,6 @@ from scrapy.utils.log import ( ) from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings -from tests.mockserver.http import MockServer -from tests.utils import get_script_run_env from tests.utils.decorators import coroutine_test, inline_callbacks_test BASE_SETTINGS: dict[str, Any] = {} @@ -764,514 +753,6 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): pytest.skip("This test is only for CrawlerRunner") -class ScriptRunnerMixin(ABC): - @property - @abstractmethod - def script_dir(self) -> Path: - raise NotImplementedError - - @staticmethod - def get_script_dir(name: str) -> Path: - return Path(__file__).parent.resolve() / name - - def get_script_args(self, script_name: str, *script_args: str) -> list[str]: - script_path = self.script_dir / script_name - return [sys.executable, str(script_path), *script_args] - - def run_script(self, script_name: str, *script_args: str) -> str: - args = self.get_script_args(script_name, *script_args) - p = subprocess.Popen( - args, - env=get_script_run_env(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - _, stderr = p.communicate() - return stderr.decode("utf-8") - - -class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): - """Common tests between CrawlerProcess and AsyncCrawlerProcess, - with the same file names and expectations. - """ - - def test_simple(self): - log = self.run_script("simple.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "is_reactorless(): False" in log - - def test_multi(self): - log = self.run_script("multi.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "ReactorAlreadyInstalledError" not in log - - def test_reactor_default(self): - log = self.run_script("reactor_default.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" - ) in log - - def test_asyncio_enabled_no_reactor(self): - log = self.run_script("asyncio_enabled_no_reactor.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "RuntimeError" not in log - - def test_asyncio_enabled_reactor(self): - log = self.run_script("asyncio_enabled_reactor.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "RuntimeError" not in log - - @pytest.mark.skipif( - parse_version(w3lib_version) >= parse_version("2.0.0"), - reason="w3lib 2.0.0 and later do not allow invalid domains.", - ) - def test_ipv6_default_name_resolver(self): - log = self.run_script("default_name_resolver.py") - assert "Spider closed (finished)" in log - assert ( - "'downloader/exception_type_count/scrapy.exceptions.CannotResolveHostError': 1," - in log - ) - assert ( - "scrapy.exceptions.CannotResolveHostError: DNS lookup failed: no results for hostname lookup: ::1." - in log - ) - - def test_caching_hostname_resolver_ipv6(self): - log = self.run_script("caching_hostname_resolver_ipv6.py") - assert "Spider closed (finished)" in log - assert "scrapy.exceptions.CannotResolveHostError" not in log - - def test_caching_hostname_resolver_finite_execution( - self, mockserver: MockServer - ) -> None: - log = self.run_script("caching_hostname_resolver.py", mockserver.url("/")) - assert "Spider closed (finished)" in log - assert "ERROR: Error downloading" not in log - assert "TimeoutError" not in log - assert "scrapy.exceptions.CannotResolveHostError" not in log - - def test_twisted_reactor_asyncio(self): - log = self.run_script("twisted_reactor_asyncio.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - def test_twisted_reactor_asyncio_custom_settings(self): - log = self.run_script("twisted_reactor_custom_settings.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - def test_twisted_reactor_asyncio_custom_settings_same(self): - log = self.run_script("twisted_reactor_custom_settings_same.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - @pytest.mark.requires_uvloop - def test_custom_loop_asyncio(self): - log = self.run_script("asyncio_custom_loop.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - @pytest.mark.requires_uvloop - def test_custom_loop_asyncio_deferred_signal(self): - log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" in log - assert "async pipeline opened!" in log - - @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_same_loop(self): - log = self.run_script("asyncio_enabled_reactor_same_loop.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_different_loop(self): - log = self.run_script("asyncio_enabled_reactor_different_loop.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the one specified in the ASYNCIO_EVENT_LOOP " - "setting (uvloop.Loop)" - ) in log - - def test_default_loop_asyncio_deferred_signal(self): - log = self.run_script("asyncio_deferred_signal.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" not in log - assert "async pipeline opened!" in log - - def test_args_change_settings(self): - log = self.run_script("args_settings.py") - assert "Spider closed (finished)" in log - assert "The value of FOO is 42" in log - - def test_shutdown_graceful(self): - sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "3") - p = PopenSpawn(args, timeout=5, env=get_script_run_env()) - p.expect_exact("Spider opened") - p.expect_exact("Crawled (200)") - p.kill(sig) - p.expect_exact("shutting down gracefully") - p.expect_exact("Spider closed (shutdown)") - p.wait() - - @inline_callbacks_test - def test_shutdown_forced(self): - sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "10") - p = PopenSpawn(args, timeout=5, env=get_script_run_env()) - p.expect_exact("Spider opened") - p.expect_exact("Crawled (200)") - p.kill(sig) - p.expect_exact("shutting down gracefully") - # sending the second signal too fast often causes problems - d = Deferred() - call_later(0.01, d.callback, None) - yield d - p.kill(sig) - p.expect_exact("forcing unclean shutdown") - p.wait() - - -class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): - @property - def script_dir(self) -> Path: - return self.get_script_dir("CrawlerProcess") - - def test_reactor_default_twisted_reactor_select(self): - log = self.run_script("reactor_default_twisted_reactor_select.py") - if platform.system() in ["Windows", "Darwin"]: - # The goal of this test function is to test that, when a reactor is - # installed (the default one here) and a different reactor is - # configured (select here), an error raises. - # - # In Windows the default reactor is the select reactor, so that - # error does not raise. - # - # If that ever becomes the case on more platforms (i.e. if Linux - # also starts using the select reactor by default in a future - # version of Twisted), then we will need to rethink this test. - assert "Spider closed (finished)" in log - else: - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - - def test_reactor_select(self): - log = self.run_script("reactor_select.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" - ) in log - - def test_reactor_select_twisted_reactor_select(self): - log = self.run_script("reactor_select_twisted_reactor_select.py") - assert "Spider closed (finished)" in log - assert "ReactorAlreadyInstalledError" not in log - - def test_reactor_select_subclass_twisted_reactor_select(self): - log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - - def test_twisted_reactor_select(self): - log = self.run_script("twisted_reactor_select.py") - assert "Spider closed (finished)" in log - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - - @pytest.mark.skipif( - platform.system() == "Windows", reason="PollReactor is not supported on Windows" - ) - def test_twisted_reactor_poll(self): - log = self.run_script("twisted_reactor_poll.py") - assert "Spider closed (finished)" in log - assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log - - def test_twisted_reactor_asyncio_custom_settings_conflict(self): - log = self.run_script("twisted_reactor_custom_settings_conflict.py") - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - assert ( - "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" - in log - ) - - def test_reactorless(self): - log = self.run_script("reactorless.py") - assert ( - "RuntimeError: CrawlerProcess doesn't support TWISTED_ENABLED=False" in log - ) - - -class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): - @property - def script_dir(self) -> Path: - return self.get_script_dir("AsyncCrawlerProcess") - - def test_twisted_reactor_custom_settings_select(self): - log = self.run_script("twisted_reactor_custom_settings_select.py") - assert "Spider closed (finished)" not in log - assert ( - "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) " - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - - @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_same_loop(self): - log = self.run_script("asyncio_custom_loop_custom_settings_same.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_different_loop(self): - log = self.run_script("asyncio_custom_loop_custom_settings_different.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the one specified in the ASYNCIO_EVENT_LOOP " - "setting (uvloop.Loop)" - ) in log - - def test_reactorless_simple(self): - log = self.run_script("reactorless_simple.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "is_reactorless(): True" in log - assert "ERROR: " not in log - assert "WARNING: " not in log - - def test_reactorless_datauri(self): - log = self.run_script("reactorless_datauri.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "{'data': 'foo'}" in log - assert "'item_scraped_count': 1" in log - assert "ERROR: " not in log - assert "WARNING: " not in log - - def test_reactorless_import_hook(self): - log = self.run_script("reactorless_import_hook.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "ImportError: Import of twisted.internet.reactor is forbidden" in log - - def test_reactorless_telnetconsole_default(self): - """By default TWISTED_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False.""" - log = self.run_script("reactorless_telnetconsole_default.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "The TelnetConsole extension requires a Twisted reactor" not in log - assert "scrapy.extensions.telnet.TelnetConsole" not in log - - def test_reactorless_telnetconsole_disabled(self): - """Explicit TELNETCONSOLE_ENABLED=False, there are no warnings.""" - log = self.run_script("reactorless_telnetconsole_disabled.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "The TelnetConsole extension requires a Twisted reactor" not in log - assert "scrapy.extensions.telnet.TelnetConsole" not in log - - def test_reactorless_telnetconsole_enabled(self): - """Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning.""" - log = self.run_script("reactorless_telnetconsole_enabled.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "The TelnetConsole extension requires a Twisted reactor" in log - - def test_reactorless_reactor(self): - log = self.run_script("reactorless_reactor.py") - assert ( - "RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed" - in log - ) - - -class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): - """Common tests between CrawlerRunner and AsyncCrawlerRunner, - with the same file names and expectations. - """ - - def test_simple(self): - log = self.run_script("simple.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "is_reactorless(): False" in log - - def test_multi_parallel(self): - log = self.run_script("multi_parallel.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert re.search( - r"Spider opened.+Spider opened.+Closing spider.+Closing spider", - log, - re.DOTALL, - ) - - def test_multi_seq(self): - log = self.run_script("multi_seq.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert re.search( - r"Spider opened.+Closing spider.+Spider opened.+Closing spider", - log, - re.DOTALL, - ) - - @pytest.mark.requires_uvloop - def test_custom_loop_same(self): - log = self.run_script("custom_loop_same.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - @pytest.mark.requires_uvloop - def test_custom_loop_different(self): - log = self.run_script("custom_loop_different.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the one specified in the ASYNCIO_EVENT_LOOP " - "setting (uvloop.Loop)" - ) in log - - -class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): - @property - def script_dir(self) -> Path: - return self.get_script_dir("CrawlerRunner") - - def test_explicit_default_reactor(self): - log = self.run_script("explicit_default_reactor.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - - def test_response_ip_address(self): - log = self.run_script("ip_address.py") - assert "INFO: Spider closed (finished)" in log - assert "INFO: Host: not.a.real.domain" in log - assert "INFO: Type: " in log - assert "INFO: IP address: 127.0.0.1" in log - - def test_change_default_reactor(self): - log = self.run_script("change_reactor.py") - assert ( - "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "DEBUG: Using asyncio event loop" in log - - def test_reactorless(self): - log = self.run_script("reactorless.py") - assert ( - "RuntimeError: CrawlerRunner doesn't support TWISTED_ENABLED=False" in log - ) - - -class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): - @property - def script_dir(self) -> Path: - return self.get_script_dir("AsyncCrawlerRunner") - - def test_simple_default_reactor(self): - log = self.run_script("simple_default_reactor.py") - assert "Spider closed (finished)" not in log - assert ( - "RuntimeError: When TWISTED_ENABLED is True, " - "AsyncCrawlerRunner requires that the installed Twisted reactor" - ) in log - - def test_reactorless_simple(self): - log = self.run_script("reactorless_simple.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "is_reactorless(): True" in log - assert "ERROR: " not in log - assert "WARNING: " not in log - - def test_reactorless_datauri(self): - log = self.run_script("reactorless_datauri.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "{'data': 'foo'}" in log - assert "'item_scraped_count': 1" in log - assert "ERROR: " not in log - assert "WARNING: " not in log - - def test_reactorless_reactor(self): - log = self.run_script("reactorless_reactor.py") - assert ( - "RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed" - in log - ) - - @pytest.mark.parametrize( ("settings", "items"), [ diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py new file mode 100644 index 000000000..8d7b8f871 --- /dev/null +++ b/tests/test_crawler_subprocess.py @@ -0,0 +1,531 @@ +from __future__ import annotations + +import platform +import re +import signal +import subprocess +import sys +from abc import ABC, abstractmethod +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +from packaging.version import parse as parse_version +from pexpect.popen_spawn import PopenSpawn +from twisted.internet.defer import Deferred +from w3lib import __version__ as w3lib_version + +from scrapy.utils.asyncio import call_later +from tests.utils import get_script_run_env +from tests.utils.decorators import inline_callbacks_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer + + +class ScriptRunnerMixin(ABC): + @property + @abstractmethod + def script_dir(self) -> Path: + raise NotImplementedError + + @staticmethod + def get_script_dir(name: str) -> Path: + return Path(__file__).parent.resolve() / name + + def get_script_args(self, script_name: str, *script_args: str) -> list[str]: + script_path = self.script_dir / script_name + return [sys.executable, str(script_path), *script_args] + + def run_script(self, script_name: str, *script_args: str) -> str: + args = self.get_script_args(script_name, *script_args) + p = subprocess.Popen( + args, + env=get_script_run_env(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + _, stderr = p.communicate() + return stderr.decode("utf-8") + + +class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): + """Common tests between CrawlerProcess and AsyncCrawlerProcess, + with the same file names and expectations. + """ + + def test_simple(self): + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "is_reactorless(): False" in log + + def test_multi(self): + log = self.run_script("multi.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "ReactorAlreadyInstalledError" not in log + + def test_reactor_default(self): + log = self.run_script("reactor_default.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log + + def test_asyncio_enabled_no_reactor(self): + log = self.run_script("asyncio_enabled_no_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "RuntimeError" not in log + + def test_asyncio_enabled_reactor(self): + log = self.run_script("asyncio_enabled_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "RuntimeError" not in log + + @pytest.mark.skipif( + parse_version(w3lib_version) >= parse_version("2.0.0"), + reason="w3lib 2.0.0 and later do not allow invalid domains.", + ) + def test_ipv6_default_name_resolver(self): + log = self.run_script("default_name_resolver.py") + assert "Spider closed (finished)" in log + assert ( + "'downloader/exception_type_count/scrapy.exceptions.CannotResolveHostError': 1," + in log + ) + assert ( + "scrapy.exceptions.CannotResolveHostError: DNS lookup failed: no results for hostname lookup: ::1." + in log + ) + + def test_caching_hostname_resolver_ipv6(self): + log = self.run_script("caching_hostname_resolver_ipv6.py") + assert "Spider closed (finished)" in log + assert "scrapy.exceptions.CannotResolveHostError" not in log + + def test_caching_hostname_resolver_finite_execution( + self, mockserver: MockServer + ) -> None: + log = self.run_script("caching_hostname_resolver.py", mockserver.url("/")) + assert "Spider closed (finished)" in log + assert "ERROR: Error downloading" not in log + assert "TimeoutError" not in log + assert "scrapy.exceptions.CannotResolveHostError" not in log + + def test_twisted_reactor_asyncio(self): + log = self.run_script("twisted_reactor_asyncio.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_twisted_reactor_asyncio_custom_settings(self): + log = self.run_script("twisted_reactor_custom_settings.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_twisted_reactor_asyncio_custom_settings_same(self): + log = self.run_script("twisted_reactor_custom_settings_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + @pytest.mark.requires_uvloop + def test_custom_loop_asyncio(self): + log = self.run_script("asyncio_custom_loop.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_custom_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + assert "async pipeline opened!" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_same_loop(self): + log = self.run_script("asyncio_enabled_reactor_same_loop.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_different_loop(self): + log = self.run_script("asyncio_enabled_reactor_different_loop.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + def test_default_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" not in log + assert "async pipeline opened!" in log + + def test_args_change_settings(self): + log = self.run_script("args_settings.py") + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log + + def test_shutdown_graceful(self): + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK + args = self.get_script_args("sleeping.py", "3") + p = PopenSpawn(args, timeout=5, env=get_script_run_env()) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + p.expect_exact("Spider closed (shutdown)") + p.wait() + + @inline_callbacks_test + def test_shutdown_forced(self): + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK + args = self.get_script_args("sleeping.py", "10") + p = PopenSpawn(args, timeout=5, env=get_script_run_env()) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + # sending the second signal too fast often causes problems + d = Deferred() + call_later(0.01, d.callback, None) + yield d + p.kill(sig) + p.expect_exact("forcing unclean shutdown") + p.wait() + + +class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerProcess") + + def test_reactor_default_twisted_reactor_select(self): + log = self.run_script("reactor_default_twisted_reactor_select.py") + if platform.system() in ["Windows", "Darwin"]: + # The goal of this test function is to test that, when a reactor is + # installed (the default one here) and a different reactor is + # configured (select here), an error raises. + # + # In Windows the default reactor is the select reactor, so that + # error does not raise. + # + # If that ever becomes the case on more platforms (i.e. if Linux + # also starts using the select reactor by default in a future + # version of Twisted), then we will need to rethink this test. + assert "Spider closed (finished)" in log + else: + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_reactor_select(self): + log = self.run_script("reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log + + def test_reactor_select_twisted_reactor_select(self): + log = self.run_script("reactor_select_twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "ReactorAlreadyInstalledError" not in log + + def test_reactor_select_subclass_twisted_reactor_select(self): + log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_twisted_reactor_select(self): + log = self.run_script("twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + + @pytest.mark.skipif( + platform.system() == "Windows", reason="PollReactor is not supported on Windows" + ) + def test_twisted_reactor_poll(self): + log = self.run_script("twisted_reactor_poll.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log + + def test_twisted_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" + in log + ) + + def test_reactorless(self): + log = self.run_script("reactorless.py") + assert ( + "RuntimeError: CrawlerProcess doesn't support TWISTED_ENABLED=False" in log + ) + + +class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerProcess") + + def test_twisted_reactor_custom_settings_select(self): + log = self.run_script("twisted_reactor_custom_settings_select.py") + assert "Spider closed (finished)" not in log + assert ( + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) " + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_same_loop(self): + log = self.run_script("asyncio_custom_loop_custom_settings_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_different_loop(self): + log = self.run_script("asyncio_custom_loop_custom_settings_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + def test_reactorless_simple(self): + log = self.run_script("reactorless_simple.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "is_reactorless(): True" in log + assert "ERROR: " not in log + assert "WARNING: " not in log + + def test_reactorless_datauri(self): + log = self.run_script("reactorless_datauri.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "{'data': 'foo'}" in log + assert "'item_scraped_count': 1" in log + assert "ERROR: " not in log + assert "WARNING: " not in log + + def test_reactorless_import_hook(self): + log = self.run_script("reactorless_import_hook.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "ImportError: Import of twisted.internet.reactor is forbidden" in log + + def test_reactorless_telnetconsole_default(self): + """By default TWISTED_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False.""" + log = self.run_script("reactorless_telnetconsole_default.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" not in log + assert "scrapy.extensions.telnet.TelnetConsole" not in log + + def test_reactorless_telnetconsole_disabled(self): + """Explicit TELNETCONSOLE_ENABLED=False, there are no warnings.""" + log = self.run_script("reactorless_telnetconsole_disabled.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" not in log + assert "scrapy.extensions.telnet.TelnetConsole" not in log + + def test_reactorless_telnetconsole_enabled(self): + """Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning.""" + log = self.run_script("reactorless_telnetconsole_enabled.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" in log + + def test_reactorless_reactor(self): + log = self.run_script("reactorless_reactor.py") + assert ( + "RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed" + in log + ) + + +class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): + """Common tests between CrawlerRunner and AsyncCrawlerRunner, + with the same file names and expectations. + """ + + def test_simple(self): + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "is_reactorless(): False" in log + + def test_multi_parallel(self): + log = self.run_script("multi_parallel.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Spider opened.+Closing spider.+Closing spider", + log, + re.DOTALL, + ) + + def test_multi_seq(self): + log = self.run_script("multi_seq.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Closing spider.+Spider opened.+Closing spider", + log, + re.DOTALL, + ) + + @pytest.mark.requires_uvloop + def test_custom_loop_same(self): + log = self.run_script("custom_loop_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_custom_loop_different(self): + log = self.run_script("custom_loop_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + +class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerRunner") + + def test_explicit_default_reactor(self): + log = self.run_script("explicit_default_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + + def test_response_ip_address(self): + log = self.run_script("ip_address.py") + assert "INFO: Spider closed (finished)" in log + assert "INFO: Host: not.a.real.domain" in log + assert "INFO: Type: " in log + assert "INFO: IP address: 127.0.0.1" in log + + def test_change_default_reactor(self): + log = self.run_script("change_reactor.py") + assert ( + "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "DEBUG: Using asyncio event loop" in log + + def test_reactorless(self): + log = self.run_script("reactorless.py") + assert ( + "RuntimeError: CrawlerRunner doesn't support TWISTED_ENABLED=False" in log + ) + + +class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerRunner") + + def test_simple_default_reactor(self): + log = self.run_script("simple_default_reactor.py") + assert "Spider closed (finished)" not in log + assert ( + "RuntimeError: When TWISTED_ENABLED is True, " + "AsyncCrawlerRunner requires that the installed Twisted reactor" + ) in log + + def test_reactorless_simple(self): + log = self.run_script("reactorless_simple.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "is_reactorless(): True" in log + assert "ERROR: " not in log + assert "WARNING: " not in log + + def test_reactorless_datauri(self): + log = self.run_script("reactorless_datauri.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "{'data': 'foo'}" in log + assert "'item_scraped_count': 1" in log + assert "ERROR: " not in log + assert "WARNING: " not in log + + def test_reactorless_reactor(self): + log = self.run_script("reactorless_reactor.py") + assert ( + "RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed" + in log + ) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index ab63ad545..42a25cd5b 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,16 +1,10 @@ import logging -from itertools import chain, product from unittest.mock import MagicMock import pytest -from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware -from scrapy.downloadermiddlewares.redirect import ( - MetaRefreshMiddleware, - RedirectMiddleware, -) -from scrapy.exceptions import IgnoreRequest -from scrapy.http import HtmlResponse, Request, Response +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware +from scrapy.http import Request, Response from scrapy.spidermiddlewares.referer import ( POLICY_NO_REFERRER, POLICY_ORIGIN, @@ -18,984 +12,14 @@ from scrapy.spidermiddlewares.referer import ( RefererMiddleware, ) from scrapy.spiders import Spider -from scrapy.utils.misc import build_from_crawler, set_environ +from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler - - -class Base: - class Test: - def test_priority_adjust(self): - req = Request("http://a.example") - rsp = self.get_response(req, "http://a.example/redirected") - req2 = self.mw.process_response(req, rsp) - assert req2.priority > req.priority - - def test_dont_redirect(self): - url = "http://www.example.com/301" - url2 = "http://www.example.com/redirected" - req = Request(url, meta={"dont_redirect": True}) - rsp = self.get_response(req, url2) - - r = self.mw.process_response(req, rsp) - assert isinstance(r, Response) - assert r is rsp - - # Test that it redirects when dont_redirect is False - req = Request(url, meta={"dont_redirect": False}) - rsp = self.get_response(req, url2) - - r = self.mw.process_response(req, rsp) - assert isinstance(r, Request) - - def test_post(self): - url = "http://www.example.com/302" - url2 = "http://www.example.com/redirected2" - req = Request( - url, - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = self.get_response(req, url2) - - req2 = self.mw.process_response(req, rsp) - assert isinstance(req2, Request) - assert req2.url == url2 - assert req2.method == "GET" - assert "Content-Type" not in req2.headers, ( - "Content-Type header must not be present in redirected request" - ) - assert "Content-Length" not in req2.headers, ( - "Content-Length header must not be present in redirected request" - ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://a.example/302") - rsp = self.get_response(req, "/redirected") - - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - assert req.meta["redirect_times"] == 1 - with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://a.example/302", meta={"redirect_ttl": 1}) - rsp = self.get_response(req, "/a") - - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp) - - def test_redirect_urls(self): - req1 = Request("http://a.example/first") - rsp1 = self.get_response(req1, "/redirected") - req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req1, "/redirected2") - req3 = self.mw.process_response(req2, rsp2) - - assert req2.url == "http://a.example/redirected" - assert req2.meta["redirect_urls"] == ["http://a.example/first"] - assert req3.url == "http://a.example/redirected2" - assert req3.meta["redirect_urls"] == [ - "http://a.example/first", - "http://a.example/redirected", - ] - - def test_redirect_reasons(self): - req1 = Request("http://a.example/first") - rsp1 = self.get_response(req1, "/redirected1") - req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req2, "/redirected2") - req3 = self.mw.process_response(req2, rsp2) - assert req2.meta["redirect_reasons"] == [self.reason] - assert req3.meta["redirect_reasons"] == [self.reason, self.reason] - - def test_cross_origin_header_dropping(self): - safe_headers = {"A": "B"} - cookie_header = {"Cookie": "a=b"} - authorization_header = {"Authorization": "Bearer 123456"} - - original_request = Request( - "https://example.com", - headers={**safe_headers, **cookie_header, **authorization_header}, - ) - - # Redirects to the same origin (same scheme, same domain, same port) - # keep all headers. - internal_response = self.get_response( - original_request, "https://example.com/a" - ) - internal_redirect_request = self.mw.process_response( - original_request, internal_response - ) - assert isinstance(internal_redirect_request, Request) - assert original_request.headers == internal_redirect_request.headers - - # Redirects to the same origin (same scheme, same domain, same port) - # keep all headers also when the scheme is http. - http_request = Request( - "http://example.com", - headers={**safe_headers, **cookie_header, **authorization_header}, - ) - http_response = self.get_response(http_request, "http://example.com/a") - http_redirect_request = self.mw.process_response( - http_request, http_response - ) - assert isinstance(http_redirect_request, Request) - assert http_request.headers == http_redirect_request.headers - - # For default ports, whether the port is explicit or implicit does not - # affect the outcome, it is still the same origin. - to_explicit_port_response = self.get_response( - original_request, "https://example.com:443/a" - ) - to_explicit_port_redirect_request = self.mw.process_response( - original_request, to_explicit_port_response - ) - assert isinstance(to_explicit_port_redirect_request, Request) - assert original_request.headers == to_explicit_port_redirect_request.headers - - # For default ports, whether the port is explicit or implicit does not - # affect the outcome, it is still the same origin. - to_implicit_port_response = self.get_response( - original_request, "https://example.com/a" - ) - to_implicit_port_redirect_request = self.mw.process_response( - original_request, to_implicit_port_response - ) - assert isinstance(to_implicit_port_redirect_request, Request) - assert original_request.headers == to_implicit_port_redirect_request.headers - - # A port change drops the Authorization header because the origin - # changes, but keeps the Cookie header because the domain remains the - # same. - different_port_response = self.get_response( - original_request, "https://example.com:8080/a" - ) - different_port_redirect_request = self.mw.process_response( - original_request, different_port_response - ) - assert isinstance(different_port_redirect_request, Request) - assert { - **safe_headers, - **cookie_header, - } == different_port_redirect_request.headers.to_unicode_dict() - - # A domain change drops both the Authorization and the Cookie header. - external_response = self.get_response( - original_request, "https://example.org/a" - ) - external_redirect_request = self.mw.process_response( - original_request, external_response - ) - assert isinstance(external_redirect_request, Request) - assert safe_headers == external_redirect_request.headers.to_unicode_dict() - - # A scheme upgrade (http → https) drops the Authorization header - # because the origin changes, but keeps the Cookie header because the - # domain remains the same. - upgrade_response = self.get_response(http_request, "https://example.com/a") - upgrade_redirect_request = self.mw.process_response( - http_request, upgrade_response - ) - assert isinstance(upgrade_redirect_request, Request) - assert { - **safe_headers, - **cookie_header, - } == upgrade_redirect_request.headers.to_unicode_dict() - - # A scheme downgrade (https → http) drops the Authorization header - # because the origin changes, and the Cookie header because its value - # cannot indicate whether the cookies were secure (HTTPS-only) or not. - # - # Note: If the Cookie header is set by the cookie management - # middleware, as recommended in the docs, the dropping of Cookie on - # scheme downgrade is not an issue, because the cookie management - # middleware will add again the Cookie header to the new request if - # appropriate. - downgrade_response = self.get_response( - original_request, "http://example.com/a" - ) - downgrade_redirect_request = self.mw.process_response( - original_request, downgrade_response - ) - assert isinstance(downgrade_redirect_request, Request) - assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() - - def test_meta_proxy_http_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_http_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_http_to_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_to_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_http_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_http_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_https_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_https_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_proxied_http_to_proxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request2.meta["_auth_proxy"] == "https://b.example" - assert request2.meta["proxy"] == "https://b.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_proxied_http_to_unproxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_unproxied_http_to_proxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request2.meta["_auth_proxy"] == "https://b.example" - assert request2.meta["proxy"] == "https://b.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_unproxied_http_to_unproxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_proxied_https_to_proxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request1.meta["_auth_proxy"] == "https://b.example" - assert request1.meta["proxy"] == "https://b.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request3.meta["_auth_proxy"] == "https://b.example" - assert request3.meta["proxy"] == "https://b.example" - - def test_system_proxy_proxied_https_to_unproxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request1.meta["_auth_proxy"] == "https://b.example" - assert request1.meta["proxy"] == "https://b.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request3.meta["_auth_proxy"] == "https://b.example" - assert request3.meta["proxy"] == "https://b.example" - - def test_system_proxy_unproxied_https_to_proxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_unproxied_https_to_unproxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta +from tests.test_downloadermiddleware_redirect_base import ( + REDIRECT_SCHEME_CASES, + SCHEME_PARAMS, + Base, +) class TestRedirectMiddleware(Base.Test): @@ -1317,44 +341,6 @@ class TestRedirectMiddleware(Base.Test): assert self.mw.process_response(request, response) is response -SCHEME_PARAMS = ("url", "location", "target") -HTTP_SCHEMES = ("http", "https") -NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo") -REDIRECT_SCHEME_CASES = ( - # http/https → http/https redirects - *( - ( - f"{input_scheme}://example.com/a", - f"{output_scheme}://example.com/b", - f"{output_scheme}://example.com/b", - ) - for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2) - ), - # http/https → data/file/ftp/s3/foo does not redirect - *( - ( - f"{input_scheme}://example.com/a", - f"{output_scheme}://example.com/b", - None, - ) - for input_scheme in HTTP_SCHEMES - for output_scheme in NON_HTTP_SCHEMES - ), - # http/https → relative redirects - *( - ( - f"{scheme}://example.com/a", - location, - f"{scheme}://example.com/b", - ) - for scheme in HTTP_SCHEMES - for location in ("//example.com/b", "/b") - ), - # Note: We do not test data/file/ftp/s3 schemes for the initial URL - # because their download handlers cannot return a status code of 3xx. -) - - @pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES) def test_redirect_schemes(url, location, target): crawler = get_crawler(Spider) @@ -1369,125 +355,6 @@ def test_redirect_schemes(url, location, target): assert redirect.url == target -def meta_refresh_body(url, interval=5): - html = f"""""" - return html.encode("utf-8") - - -class TestMetaRefreshMiddleware(Base.Test): - mwcls = MetaRefreshMiddleware - reason = "meta refresh" - - def setup_method(self): - crawler = get_crawler(Spider) - self.mw = self.mwcls.from_crawler(crawler) - - def _body(self, interval=5, url="http://example.org/newpage"): - return meta_refresh_body(url, interval) - - def get_response(self, request, location): - return HtmlResponse(request.url, body=self._body(url=location)) - - def test_meta_refresh(self): - req = Request(url="http://example.org") - rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp) - assert isinstance(req2, Request) - assert req2.url == "http://example.org/newpage" - - def test_meta_refresh_with_high_interval(self): - # meta-refresh with high intervals don't trigger redirects - req = Request(url="http://example.org") - rsp = HtmlResponse( - url="http://example.org", body=self._body(interval=1000), encoding="utf-8" - ) - rsp2 = self.mw.process_response(req, rsp) - assert rsp is rsp2 - - def test_meta_refresh_trough_posted_request(self): - req = Request( - url="http://example.org", - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp) - - assert isinstance(req2, Request) - assert req2.url == "http://example.org/newpage" - assert req2.method == "GET" - assert "Content-Type" not in req2.headers, ( - "Content-Type header must not be present in redirected request" - ) - assert "Content-Length" not in req2.headers, ( - "Content-Length header must not be present in redirected request" - ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - def test_ignore_tags_default(self): - req = Request(url="http://example.org") - body = ( - """""" - ) - rsp = HtmlResponse(req.url, body=body.encode()) - response = self.mw.process_response(req, rsp) - assert isinstance(response, Response) - - def test_ignore_tags_1_x_list(self): - """Test that Scrapy 1.x behavior remains possible""" - settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]} - crawler = get_crawler(Spider, settings) - mw = MetaRefreshMiddleware.from_crawler(crawler) - req = Request(url="http://example.org") - body = ( - """""" - ) - rsp = HtmlResponse(req.url, body=body.encode()) - response = mw.process_response(req, rsp) - assert isinstance(response, Response) - - -@pytest.mark.parametrize( - SCHEME_PARAMS, - [ - *REDIRECT_SCHEME_CASES, - # data/file/ftp/s3/foo → * does not redirect - *( - ( - f"{input_scheme}://example.com/a", - f"{output_scheme}://example.com/b", - None, - ) - for input_scheme in NON_HTTP_SCHEMES - for output_scheme in chain(HTTP_SCHEMES, NON_HTTP_SCHEMES) - ), - # data/file/ftp/s3/foo → relative does not redirect - *( - ( - f"{scheme}://example.com/a", - location, - None, - ) - for scheme in NON_HTTP_SCHEMES - for location in ("//example.com/b", "/b") - ), - ], -) -def test_meta_refresh_schemes(url, location, target): - crawler = get_crawler(Spider) - mw = MetaRefreshMiddleware.from_crawler(crawler) - request = Request(url) - response = HtmlResponse(url, body=meta_refresh_body(location)) - redirect = mw.process_response(request, response) - if target is None: - assert redirect == response - else: - assert isinstance(redirect, Request) - - @pytest.mark.parametrize( ("policy", "source_url", "target_url", "expected_referrer"), [ @@ -1569,26 +436,6 @@ def test_warning_redirect_middleware(caplog): ) in caplog.text -def test_warning_meta_refresh_middleware(caplog): - crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) - mw = build_from_crawler(MetaRefreshMiddleware, crawler) - with caplog.at_level(logging.WARNING): - mw._engine_started() - assert ( - "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware found no " - "scrapy.spidermiddlewares.referer.RefererMiddleware" - ) in caplog.text - assert ( - "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" - in caplog.text - ) - assert ( - "replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware " - "with a subclass that overrides the handle_referer() method" - ) in caplog.text - - def test_warning_subclass(caplog): class MyRedirectMiddleware(RedirectMiddleware): pass diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py new file mode 100644 index 000000000..44ade93b7 --- /dev/null +++ b/tests/test_downloadermiddleware_redirect_base.py @@ -0,0 +1,1023 @@ +from __future__ import annotations + +from itertools import product + +import pytest + +from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.http import Request, Response +from scrapy.utils.misc import set_environ +from scrapy.utils.test import get_crawler + +SCHEME_PARAMS = ("url", "location", "target") +HTTP_SCHEMES = ("http", "https") +NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo") +REDIRECT_SCHEME_CASES = ( + # http/https → http/https redirects + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + f"{output_scheme}://example.com/b", + ) + for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2) + ), + # http/https → data/file/ftp/s3/foo does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in HTTP_SCHEMES + for output_scheme in NON_HTTP_SCHEMES + ), + # http/https → relative redirects + *( + ( + f"{scheme}://example.com/a", + location, + f"{scheme}://example.com/b", + ) + for scheme in HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + # Note: We do not test data/file/ftp/s3 schemes for the initial URL + # because their download handlers cannot return a status code of 3xx. +) + + +class Base: + class Test: + def test_priority_adjust(self): + req = Request("http://a.example") + rsp = self.get_response(req, "http://a.example/redirected") + req2 = self.mw.process_response(req, rsp) + assert req2.priority > req.priority + + def test_dont_redirect(self): + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" + req = Request(url, meta={"dont_redirect": True}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Response) + assert r is rsp + + # Test that it redirects when dont_redirect is False + req = Request(url, meta={"dont_redirect": False}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Request) + + def test_post(self): + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request( + url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = self.get_response(req, url2) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + + def test_max_redirect_times(self): + self.mw.max_redirect_times = 1 + req = Request("http://a.example/302") + rsp = self.get_response(req, "/redirected") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + assert "redirect_times" in req.meta + assert req.meta["redirect_times"] == 1 + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_ttl(self): + self.mw.max_redirect_times = 100 + req = Request("http://a.example/302", meta={"redirect_ttl": 1}) + rsp = self.get_response(req, "/a") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_redirect_urls(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req1, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + + assert req2.url == "http://a.example/redirected" + assert req2.meta["redirect_urls"] == ["http://a.example/first"] + assert req3.url == "http://a.example/redirected2" + assert req3.meta["redirect_urls"] == [ + "http://a.example/first", + "http://a.example/redirected", + ] + + def test_redirect_reasons(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected1") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + assert req2.meta["redirect_reasons"] == [self.reason] + assert req3.meta["redirect_reasons"] == [self.reason, self.reason] + + def test_cross_origin_header_dropping(self): + safe_headers = {"A": "B"} + cookie_header = {"Cookie": "a=b"} + authorization_header = {"Authorization": "Bearer 123456"} + + original_request = Request( + "https://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers. + internal_response = self.get_response( + original_request, "https://example.com/a" + ) + internal_redirect_request = self.mw.process_response( + original_request, internal_response + ) + assert isinstance(internal_redirect_request, Request) + assert original_request.headers == internal_redirect_request.headers + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers also when the scheme is http. + http_request = Request( + "http://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + http_response = self.get_response(http_request, "http://example.com/a") + http_redirect_request = self.mw.process_response( + http_request, http_response + ) + assert isinstance(http_redirect_request, Request) + assert http_request.headers == http_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_explicit_port_response = self.get_response( + original_request, "https://example.com:443/a" + ) + to_explicit_port_redirect_request = self.mw.process_response( + original_request, to_explicit_port_response + ) + assert isinstance(to_explicit_port_redirect_request, Request) + assert original_request.headers == to_explicit_port_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_implicit_port_response = self.get_response( + original_request, "https://example.com/a" + ) + to_implicit_port_redirect_request = self.mw.process_response( + original_request, to_implicit_port_response + ) + assert isinstance(to_implicit_port_redirect_request, Request) + assert original_request.headers == to_implicit_port_redirect_request.headers + + # A port change drops the Authorization header because the origin + # changes, but keeps the Cookie header because the domain remains the + # same. + different_port_response = self.get_response( + original_request, "https://example.com:8080/a" + ) + different_port_redirect_request = self.mw.process_response( + original_request, different_port_response + ) + assert isinstance(different_port_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == different_port_redirect_request.headers.to_unicode_dict() + + # A domain change drops both the Authorization and the Cookie header. + external_response = self.get_response( + original_request, "https://example.org/a" + ) + external_redirect_request = self.mw.process_response( + original_request, external_response + ) + assert isinstance(external_redirect_request, Request) + assert safe_headers == external_redirect_request.headers.to_unicode_dict() + + # A scheme upgrade (http → https) drops the Authorization header + # because the origin changes, but keeps the Cookie header because the + # domain remains the same. + upgrade_response = self.get_response(http_request, "https://example.com/a") + upgrade_redirect_request = self.mw.process_response( + http_request, upgrade_response + ) + assert isinstance(upgrade_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == upgrade_redirect_request.headers.to_unicode_dict() + + # A scheme downgrade (https → http) drops the Authorization header + # because the origin changes, and the Cookie header because its value + # cannot indicate whether the cookies were secure (HTTPS-only) or not. + # + # Note: If the Cookie header is set by the cookie management + # middleware, as recommended in the docs, the dropping of Cookie on + # scheme downgrade is not an issue, because the cookie management + # middleware will add again the Cookie header to the new request if + # appropriate. + downgrade_response = self.get_response( + original_request, "http://example.com/a" + ) + downgrade_redirect_request = self.mw.process_response( + original_request, downgrade_response + ) + assert isinstance(downgrade_redirect_request, Request) + assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() + + def test_meta_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_to_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_to_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_unproxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_proxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_proxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_unproxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py new file mode 100644 index 000000000..416fbc2aa --- /dev/null +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import logging +from itertools import chain +from unittest.mock import MagicMock + +import pytest + +from scrapy.downloadermiddlewares.redirect import MetaRefreshMiddleware +from scrapy.http import HtmlResponse, Request, Response +from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler +from tests.test_downloadermiddleware_redirect_base import ( + HTTP_SCHEMES, + NON_HTTP_SCHEMES, + REDIRECT_SCHEME_CASES, + SCHEME_PARAMS, + Base, +) + + +def meta_refresh_body(url, interval=5): + html = f"""""" + return html.encode("utf-8") + + +class TestMetaRefreshMiddleware(Base.Test): + mwcls = MetaRefreshMiddleware + reason = "meta refresh" + + def setup_method(self): + crawler = get_crawler(Spider) + self.mw = self.mwcls.from_crawler(crawler) + + def _body(self, interval=5, url="http://example.org/newpage"): + return meta_refresh_body(url, interval) + + def get_response(self, request, location): + return HtmlResponse(request.url, body=self._body(url=location)) + + def test_meta_refresh(self): + req = Request(url="http://example.org") + rsp = HtmlResponse(req.url, body=self._body()) + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == "http://example.org/newpage" + + def test_meta_refresh_with_high_interval(self): + # meta-refresh with high intervals don't trigger redirects + req = Request(url="http://example.org") + rsp = HtmlResponse( + url="http://example.org", body=self._body(interval=1000), encoding="utf-8" + ) + rsp2 = self.mw.process_response(req, rsp) + assert rsp is rsp2 + + def test_meta_refresh_trough_posted_request(self): + req = Request( + url="http://example.org", + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = HtmlResponse(req.url, body=self._body()) + req2 = self.mw.process_response(req, rsp) + + assert isinstance(req2, Request) + assert req2.url == "http://example.org/newpage" + assert req2.method == "GET" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + + def test_ignore_tags_default(self): + req = Request(url="http://example.org") + body = ( + """""" + ) + rsp = HtmlResponse(req.url, body=body.encode()) + response = self.mw.process_response(req, rsp) + assert isinstance(response, Response) + + def test_ignore_tags_1_x_list(self): + """Test that Scrapy 1.x behavior remains possible""" + settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]} + crawler = get_crawler(Spider, settings) + mw = MetaRefreshMiddleware.from_crawler(crawler) + req = Request(url="http://example.org") + body = ( + """""" + ) + rsp = HtmlResponse(req.url, body=body.encode()) + response = mw.process_response(req, rsp) + assert isinstance(response, Response) + + +@pytest.mark.parametrize( + SCHEME_PARAMS, + [ + *REDIRECT_SCHEME_CASES, + # data/file/ftp/s3/foo → * does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in NON_HTTP_SCHEMES + for output_scheme in chain(HTTP_SCHEMES, NON_HTTP_SCHEMES) + ), + # data/file/ftp/s3/foo → relative does not redirect + *( + ( + f"{scheme}://example.com/a", + location, + None, + ) + for scheme in NON_HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + ], +) +def test_meta_refresh_schemes(url, location, target): + crawler = get_crawler(Spider) + mw = MetaRefreshMiddleware.from_crawler(crawler) + request = Request(url) + response = HtmlResponse(url, body=meta_refresh_body(location)) + redirect = mw.process_response(request, response) + if target is None: + assert redirect == response + else: + assert isinstance(redirect, Request) + + +def test_warning_meta_refresh_middleware(caplog): + crawler = get_crawler() + crawler.get_spider_middleware = MagicMock(return_value=None) + mw = build_from_crawler(MetaRefreshMiddleware, crawler) + with caplog.at_level(logging.WARNING): + mw._engine_started() + assert ( + "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware found no " + "scrapy.spidermiddlewares.referer.RefererMiddleware" + ) in caplog.text + assert ( + "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" + in caplog.text + ) + assert ( + "replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware " + "with a subclass that overrides the handle_referer() method" + ) in caplog.text diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1ca5f21f0..bd6ac5967 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,65 +1,47 @@ from __future__ import annotations -import bz2 import csv -import gzip import json -import lzma import marshal -import os import pickle import random import shutil -import string -import sys import tempfile -import warnings from abc import ABC, abstractmethod -from collections import defaultdict -from io import BytesIO from logging import getLogger from pathlib import Path from string import ascii_letters, digits -from typing import IO, TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any from unittest import mock -from urllib.parse import quote, urljoin +from urllib.parse import urljoin from urllib.request import pathname2url import lxml.etree import pytest -from packaging.version import Version from testfixtures import LogCapture -from w3lib.url import file_uri_to_path, path_to_file_uri +from w3lib.url import file_uri_to_path from zope.interface import implementer -from zope.interface.verify import verifyObject import scrapy from scrapy import Spider, signals -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, FeedExporter, FeedSlot, FileFeedStorage, - FTPFeedStorage, - GCSFeedStorage, IFeedStorage, S3FeedStorage, - StdoutFeedStorage, ) -from scrapy.settings import Settings -from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler -from tests.mockserver.ftp import MockFTPServer from tests.mockserver.http import MockServer from tests.spiders import ItemSpider from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: from collections.abc import Callable, Iterable - from os import PathLike def path_to_url(path): @@ -70,541 +52,6 @@ def printf_escape(string): return string.replace("%", "%%") -def build_url(path: str | PathLike) -> str: - path_str = str(path) - if path_str[0] != "/": - path_str = "/" + path_str - return urljoin("file:", path_str) - - -def mock_google_cloud_storage() -> tuple[Any, Any, Any]: - """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob - classes and set their proper return values. - """ - from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415 - - client_mock = mock.create_autospec(Client) - - bucket_mock = mock.create_autospec(Bucket) - client_mock.get_bucket.return_value = bucket_mock - - blob_mock = mock.create_autospec(Blob) - bucket_mock.blob.return_value = blob_mock - - return (client_mock, bucket_mock, blob_mock) - - -class TestFileFeedStorage: - def test_store_file_uri(self, tmp_path): - path = tmp_path / "file.txt" - uri = path_to_file_uri(str(path)) - self._assert_stores(FileFeedStorage(uri), path) - - def test_store_file_uri_makedirs(self, tmp_path): - path = tmp_path / "more" / "paths" / "file.txt" - uri = path_to_file_uri(str(path)) - self._assert_stores(FileFeedStorage(uri), path) - - def test_store_direct_path(self, tmp_path): - path = tmp_path / "file.txt" - self._assert_stores(FileFeedStorage(str(path)), path) - - def test_store_direct_path_relative(self, tmp_path): - old_cwd = Path.cwd() - try: - os.chdir(tmp_path) - path = Path("foo", "bar") - self._assert_stores(FileFeedStorage(str(path)), path) - finally: - os.chdir(old_cwd) - - def test_interface(self, tmp_path): - path = tmp_path / "file.txt" - st = FileFeedStorage(str(path)) - verifyObject(IFeedStorage, st) - - @staticmethod - def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None: - storage = FileFeedStorage(str(path), feed_options=feed_options) - spider = scrapy.Spider("default") - file = storage.open(spider) - file.write(b"content") - storage.store(file) - - def test_append(self, tmp_path): - path = tmp_path / "file.txt" - self._store(path) - self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") - - def test_overwrite(self, tmp_path): - path = tmp_path / "file.txt" - self._store(path, {"overwrite": True}) - self._assert_stores( - FileFeedStorage(str(path), feed_options={"overwrite": True}), path - ) - - @staticmethod - def _assert_stores( - storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" - ) -> None: - spider = scrapy.Spider("default") - file = storage.open(spider) - file.write(b"content") - storage.store(file) - assert path.exists() - try: - assert path.read_bytes() == expected_content - finally: - path.unlink() - - def test_preserves_windows_path_without_file_scheme(self): - path = r"C:\Users\user\Desktop\test.txt" - storage = FileFeedStorage(path) - assert storage.path == path - - -@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage -class TestFTPFeedStorage: - def get_test_spider(self, settings=None): - class TestSpider(scrapy.Spider): - name = "test_spider" - - crawler = get_crawler(settings_dict=settings) - return TestSpider.from_crawler(crawler) - - async def _store(self, uri, content, feed_options=None, settings=None): - crawler = get_crawler(settings_dict=settings or {}) - storage = FTPFeedStorage.from_crawler( - crawler, - uri, - feed_options=feed_options, - ) - verifyObject(IFeedStorage, storage) - spider = self.get_test_spider() - file = storage.open(spider) - file.write(content) - await maybe_deferred_to_future(storage.store(file)) - - def _assert_stored(self, path: Path, content): - assert path.exists() - try: - assert path.read_bytes() == content - finally: - path.unlink() - - @coroutine_test - async def test_append(self): - with MockFTPServer() as ftp_server: - filename = "file" - url = ftp_server.url(filename) - feed_options = {"overwrite": False} - await self._store(url, b"foo", feed_options=feed_options) - await self._store(url, b"bar", feed_options=feed_options) - self._assert_stored(ftp_server.path / filename, b"foobar") - - @coroutine_test - async def test_overwrite(self): - with MockFTPServer() as ftp_server: - filename = "file" - url = ftp_server.url(filename) - await self._store(url, b"foo") - await self._store(url, b"bar") - self._assert_stored(ftp_server.path / filename, b"bar") - - @coroutine_test - async def test_append_active_mode(self): - with MockFTPServer() as ftp_server: - settings = {"FEED_STORAGE_FTP_ACTIVE": True} - filename = "file" - url = ftp_server.url(filename) - feed_options = {"overwrite": False} - await self._store(url, b"foo", feed_options=feed_options, settings=settings) - await self._store(url, b"bar", feed_options=feed_options, settings=settings) - self._assert_stored(ftp_server.path / filename, b"foobar") - - @coroutine_test - async def test_overwrite_active_mode(self): - with MockFTPServer() as ftp_server: - settings = {"FEED_STORAGE_FTP_ACTIVE": True} - filename = "file" - url = ftp_server.url(filename) - await self._store(url, b"foo", settings=settings) - await self._store(url, b"bar", settings=settings) - self._assert_stored(ftp_server.path / filename, b"bar") - - def test_uri_auth_quote(self): - # RFC3986: 3.2.1. User Information - pw_quoted = quote(string.punctuation, safe="") - st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) - assert st.password == string.punctuation - - -class MyBlockingFeedStorage(BlockingFeedStorage): - def _store_in_thread(self, file: IO[bytes]) -> None: - return - - -class TestBlockingFeedStorage: - def get_test_spider(self, settings=None): - class TestSpider(scrapy.Spider): - name = "test_spider" - - crawler = get_crawler(settings_dict=settings) - return TestSpider.from_crawler(crawler) - - def test_default_temp_dir(self): - b = MyBlockingFeedStorage() - - storage_file = b.open(self.get_test_spider()) - storage_dir = Path(storage_file.name).parent - assert str(storage_dir) == tempfile.gettempdir() - - def test_temp_file(self, tmp_path): - b = MyBlockingFeedStorage() - - spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) - storage_file = b.open(spider) - storage_dir = Path(storage_file.name).parent - assert storage_dir == tmp_path - - def test_invalid_folder(self, tmp_path): - b = MyBlockingFeedStorage() - - invalid_path = tmp_path / "invalid_path" - spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) - - with pytest.raises(OSError, match="Not a Directory:"): - b.open(spider=spider) - - -@pytest.mark.requires_boto3 -@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage -class TestS3FeedStorage: - def test_parse_credentials(self): - aws_credentials = { - "AWS_ACCESS_KEY_ID": "settings_key", - "AWS_SECRET_ACCESS_KEY": "settings_secret", - "AWS_SESSION_TOKEN": "settings_token", - } - crawler = get_crawler(settings_dict=aws_credentials) - # Instantiate with crawler - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - assert storage.access_key == "settings_key" - assert storage.secret_key == "settings_secret" - assert storage.session_token == "settings_token" - # Instantiate directly - storage = S3FeedStorage( - "s3://mybucket/export.csv", - aws_credentials["AWS_ACCESS_KEY_ID"], - aws_credentials["AWS_SECRET_ACCESS_KEY"], - session_token=aws_credentials["AWS_SESSION_TOKEN"], - ) - assert storage.access_key == "settings_key" - assert storage.secret_key == "settings_secret" - assert storage.session_token == "settings_token" - # URI priority > settings priority - storage = S3FeedStorage( - "s3://uri_key:uri_secret@mybucket/export.csv", - aws_credentials["AWS_ACCESS_KEY_ID"], - aws_credentials["AWS_SECRET_ACCESS_KEY"], - ) - assert storage.access_key == "uri_key" - assert storage.secret_key == "uri_secret" - - @coroutine_test - async def test_store(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - bucket = "mybucket" - key = "export.csv" - storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}") - verifyObject(IFeedStorage, storage) - - file = mock.MagicMock() - - storage.s3_client = mock.MagicMock() - await maybe_deferred_to_future(storage.store(file)) - assert storage.s3_client.upload_fileobj.call_args == mock.call( - Bucket=bucket, Key=key, Fileobj=file - ) - - def test_init_without_acl(self): - storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl is None - - def test_init_with_acl(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl == "custom-acl" - - def test_init_with_endpoint_url(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - endpoint_url="https://example.com", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.endpoint_url == "https://example.com" - - def test_init_with_region_name(self): - region_name = "ap-east-1" - storage = S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - region_name=region_name, - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.region_name == region_name - assert storage.s3_client._client_config.region_name == region_name - - def test_from_crawler_without_acl(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl is None - - def test_without_endpoint_url(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.endpoint_url is None - - def test_without_region_name(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.s3_client._client_config.region_name == "us-east-1" - - def test_from_crawler_with_acl(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "FEED_STORAGE_S3_ACL": "custom-acl", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl == "custom-acl" - - def test_from_crawler_with_endpoint_url(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "AWS_ENDPOINT_URL": "https://example.com", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.endpoint_url == "https://example.com" - - def test_from_crawler_with_region_name(self): - region_name = "ap-east-1" - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "AWS_REGION_NAME": region_name, - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.region_name == region_name - assert storage.s3_client._client_config.region_name == region_name - - @coroutine_test - async def test_store_without_acl(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl is None - - storage.s3_client = mock.MagicMock() - await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) - acl = ( - storage.s3_client.upload_fileobj.call_args[1] - .get("ExtraArgs", {}) - .get("ACL") - ) - assert acl is None - - @coroutine_test - async def test_store_with_acl(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl == "custom-acl" - - storage.s3_client = mock.MagicMock() - await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) - acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] - assert acl == "custom-acl" - - def test_overwrite_default(self): - with LogCapture() as log: - S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - assert "S3 does not support appending to files" not in str(log) - - def test_overwrite_false(self): - with LogCapture() as log: - S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - "custom-acl", - feed_options={"overwrite": False}, - ) - assert "S3 does not support appending to files" in str(log) - - -@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage -class TestGCSFeedStorage: - def test_parse_settings(self): - try: - from google.cloud.storage import Client # noqa: F401,PLC0415 - except ImportError: - pytest.skip("GCSFeedStorage requires google-cloud-storage") - - settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} - crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") - assert storage.project_id == "123" - assert storage.acl == "publicRead" - assert storage.bucket_name == "mybucket" - assert storage.blob_name == "export.csv" - - def test_parse_empty_acl(self): - try: - from google.cloud.storage import Client # noqa: F401,PLC0415 - except ImportError: - pytest.skip("GCSFeedStorage requires google-cloud-storage") - - settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} - crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") - assert storage.acl is None - - settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None} - crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") - assert storage.acl is None - - @coroutine_test - async def test_store(self): - try: - from google.cloud.storage import Client # noqa: F401,PLC0415 - except ImportError: - pytest.skip("GCSFeedStorage requires google-cloud-storage") - - uri = "gs://mybucket/export.csv" - project_id = "myproject-123" - acl = "publicRead" - (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() - with mock.patch("google.cloud.storage.Client") as m: - m.return_value = client_mock - - f = mock.Mock() - storage = GCSFeedStorage(uri, project_id, acl) - await maybe_deferred_to_future(storage.store(f)) - - f.seek.assert_called_once_with(0) - m.assert_called_once_with(project=project_id) - client_mock.get_bucket.assert_called_once_with("mybucket") - bucket_mock.blob.assert_called_once_with("export.csv") - blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) - - def test_overwrite_default(self): - with LogCapture() as log: - GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") - assert "GCS does not support appending to files" not in str(log) - - def test_overwrite_false(self): - with LogCapture() as log: - GCSFeedStorage( - "gs://mybucket/export.csv", - "myproject-123", - "custom-acl", - feed_options={"overwrite": False}, - ) - assert "GCS does not support appending to files" in str(log) - - -class TestStdoutFeedStorage: - def test_store(self): - out = BytesIO() - storage = StdoutFeedStorage("stdout:", _stdout=out) - file = storage.open(scrapy.Spider("default")) - file.write(b"content") - storage.store(file) - assert out.getvalue() == b"content" - - def test_overwrite_default(self): - with LogCapture() as log: - StdoutFeedStorage("stdout:") - assert ( - "Standard output (stdout) storage does not support overwriting" - not in str(log) - ) - - def test_overwrite_true(self): - with LogCapture() as log: - StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) - assert "Standard output (stdout) storage does not support overwriting" in str( - log - ) - - class FromCrawlerMixin: init_with_crawler = False @@ -1834,945 +1281,6 @@ class TestFeedExport(TestFeedExportBase): assert not Storage.file_was_closed -class TestFeedPostProcessedExports(TestFeedExportBase): - items = [{"foo": "bar"}] - expected = b"foo\r\nbar\r\n" - - class MyPlugin1: - def __init__(self, file, feed_options): - self.file = file - self.feed_options = feed_options - self.char = self.feed_options.get("plugin1_char", b"") - - def write(self, data): - written_count = self.file.write(data) - written_count += self.file.write(self.char) - return written_count - - def close(self): - self.file.close() - - def _named_tempfile(self, name) -> str: - return str(Path(self.temp_dir, name)) - - async def run_and_export( - self, spider_cls: type[Spider], settings: dict[str, Any] - ) -> dict[str, bytes | None]: - """Run spider with specified settings; return exported data with filename.""" - - FEEDS = settings.get("FEEDS") or {} - settings["FEEDS"] = { - printf_escape(path_to_url(file_path)): feed_options - for file_path, feed_options in FEEDS.items() - } - - content: dict[str, bytes | None] = {} - try: - spider_cls.start_urls = [self.mockserver.url("/")] - crawler = get_crawler(spider_cls, settings) - await crawler.crawl_async() - - for file_path in FEEDS: - content[str(file_path)] = ( - Path(file_path).read_bytes() if Path(file_path).exists() else None - ) - - finally: - for file_path in FEEDS: - if not Path(file_path).exists(): - continue - - Path(file_path).unlink() - - return content - - def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""): - data_stream = BytesIO() - gzipf = gzip.GzipFile( - fileobj=data_stream, - filename=filename, - mtime=mtime, - compresslevel=compresslevel, - mode="wb", - ) - gzipf.write(data) - gzipf.close() - data_stream.seek(0) - return data_stream.read() - - @coroutine_test - async def test_gzip_plugin(self): - filename = self._named_tempfile("gzip_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - }, - }, - } - - data = await self.exported_data(self.items, settings) - try: - gzip.decompress(data[filename]) - except OSError: - pytest.fail("Received invalid gzip data.") - - @coroutine_test - async def test_gzip_plugin_compresslevel(self): - filename_to_compressed = { - self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( - self.expected, compresslevel=0 - ), - self._named_tempfile("compresslevel_9"): self.get_gzip_compressed( - self.expected, compresslevel=9 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("compresslevel_0"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_compresslevel": 0, - "gzip_mtime": 0, - "gzip_filename": "", - }, - self._named_tempfile("compresslevel_9"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_compresslevel": 9, - "gzip_mtime": 0, - "gzip_filename": "", - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = gzip.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_gzip_plugin_mtime(self): - filename_to_compressed = { - self._named_tempfile("mtime_123"): self.get_gzip_compressed( - self.expected, mtime=123 - ), - self._named_tempfile("mtime_123456789"): self.get_gzip_compressed( - self.expected, mtime=123456789 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("mtime_123"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 123, - "gzip_filename": "", - }, - self._named_tempfile("mtime_123456789"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 123456789, - "gzip_filename": "", - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = gzip.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_gzip_plugin_filename(self): - filename_to_compressed = { - self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( - self.expected, filename="FILE1" - ), - self._named_tempfile("filename_FILE2"): self.get_gzip_compressed( - self.expected, filename="FILE2" - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("filename_FILE1"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 0, - "gzip_filename": "FILE1", - }, - self._named_tempfile("filename_FILE2"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 0, - "gzip_filename": "FILE2", - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = gzip.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_lzma_plugin(self): - filename = self._named_tempfile("lzma_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - }, - }, - } - - data = await self.exported_data(self.items, settings) - try: - lzma.decompress(data[filename]) - except lzma.LZMAError: - pytest.fail("Received invalid lzma data.") - - @coroutine_test - async def test_lzma_plugin_format(self): - filename_to_compressed = { - self._named_tempfile("format_FORMAT_XZ"): lzma.compress( - self.expected, format=lzma.FORMAT_XZ - ), - self._named_tempfile("format_FORMAT_ALONE"): lzma.compress( - self.expected, format=lzma.FORMAT_ALONE - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("format_FORMAT_XZ"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_format": lzma.FORMAT_XZ, - }, - self._named_tempfile("format_FORMAT_ALONE"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_format": lzma.FORMAT_ALONE, - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = lzma.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_lzma_plugin_check(self): - filename_to_compressed = { - self._named_tempfile("check_CHECK_NONE"): lzma.compress( - self.expected, check=lzma.CHECK_NONE - ), - self._named_tempfile("check_CHECK_CRC256"): lzma.compress( - self.expected, check=lzma.CHECK_SHA256 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("check_CHECK_NONE"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_check": lzma.CHECK_NONE, - }, - self._named_tempfile("check_CHECK_CRC256"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_check": lzma.CHECK_SHA256, - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = lzma.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_lzma_plugin_preset(self): - filename_to_compressed = { - self._named_tempfile("preset_PRESET_0"): lzma.compress( - self.expected, preset=0 - ), - self._named_tempfile("preset_PRESET_9"): lzma.compress( - self.expected, preset=9 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("preset_PRESET_0"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_preset": 0, - }, - self._named_tempfile("preset_PRESET_9"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_preset": 9, - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = lzma.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_lzma_plugin_filters(self): - if "PyPy" in sys.version: - # https://foss.heptapod.net/pypy/pypy/-/issues/3527 - pytest.skip("lzma filters doesn't work in PyPy") - - filters = [{"id": lzma.FILTER_LZMA2}] - compressed = lzma.compress(self.expected, filters=filters) - filename = self._named_tempfile("filters") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_filters": filters, - }, - }, - } - - data = await self.exported_data(self.items, settings) - assert compressed == data[filename] - result = lzma.decompress(data[filename]) - assert result == self.expected - - @coroutine_test - async def test_bz2_plugin(self): - filename = self._named_tempfile("bz2_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], - }, - }, - } - - data = await self.exported_data(self.items, settings) - try: - bz2.decompress(data[filename]) - except OSError: - pytest.fail("Received invalid bz2 data.") - - @coroutine_test - async def test_bz2_plugin_compresslevel(self): - filename_to_compressed = { - self._named_tempfile("compresslevel_1"): bz2.compress( - self.expected, compresslevel=1 - ), - self._named_tempfile("compresslevel_9"): bz2.compress( - self.expected, compresslevel=9 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("compresslevel_1"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], - "bz2_compresslevel": 1, - }, - self._named_tempfile("compresslevel_9"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], - "bz2_compresslevel": 9, - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = bz2.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_custom_plugin(self): - filename = self._named_tempfile("csv_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": [self.MyPlugin1], - }, - }, - } - - data = await self.exported_data(self.items, settings) - assert data[filename] == self.expected - - @coroutine_test - async def test_custom_plugin_with_parameter(self): - expected = b"foo\r\n\nbar\r\n\n" - filename = self._named_tempfile("newline") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": [self.MyPlugin1], - "plugin1_char": b"\n", - }, - }, - } - - data = await self.exported_data(self.items, settings) - assert data[filename] == expected - - @coroutine_test - async def test_custom_plugin_with_compression(self): - expected = b"foo\r\n\nbar\r\n\n" - - filename_to_decompressor = { - self._named_tempfile("bz2"): bz2.decompress, - self._named_tempfile("lzma"): lzma.decompress, - self._named_tempfile("gzip"): gzip.decompress, - } - - settings = { - "FEEDS": { - self._named_tempfile("bz2"): { - "format": "csv", - "postprocessing": [ - self.MyPlugin1, - "scrapy.extensions.postprocessing.Bz2Plugin", - ], - "plugin1_char": b"\n", - }, - self._named_tempfile("lzma"): { - "format": "csv", - "postprocessing": [ - self.MyPlugin1, - "scrapy.extensions.postprocessing.LZMAPlugin", - ], - "plugin1_char": b"\n", - }, - self._named_tempfile("gzip"): { - "format": "csv", - "postprocessing": [ - self.MyPlugin1, - "scrapy.extensions.postprocessing.GzipPlugin", - ], - "plugin1_char": b"\n", - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, decompressor in filename_to_decompressor.items(): - result = decompressor(data[filename]) - assert result == expected - - @coroutine_test - async def test_exports_compatibility_with_postproc(self): - filename_to_expected = { - self._named_tempfile("csv"): b"foo\r\nbar\r\n", - self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]', - self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n', - self._named_tempfile("xml"): b'\n' - b"\nbar\n", - } - - settings = { - "FEEDS": { - self._named_tempfile("csv"): { - "format": "csv", - "postprocessing": [self.MyPlugin1], - # empty plugin to activate postprocessing.PostProcessingManager - }, - self._named_tempfile("json"): { - "format": "json", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("jsonlines"): { - "format": "jsonlines", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("xml"): { - "format": "xml", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("marshal"): { - "format": "marshal", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("pickle"): { - "format": "pickle", - "postprocessing": [self.MyPlugin1], - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, result in data.items(): - if "pickle" in filename: - expected, result = self.items[0], pickle.loads(result) - elif "marshal" in filename: - expected, result = self.items[0], marshal.loads(result) - else: - expected = filename_to_expected[filename] - assert result == expected - - -class TestBatchDeliveries(TestFeedExportBase): - _file_mark = "_%(batch_time)s_#%(batch_id)02d_" - - async def run_and_export( - self, spider_cls: type[Spider], settings: dict[str, Any] - ) -> dict[str, list[bytes]]: - """Run spider with specified settings; return exported data.""" - - FEEDS = settings.get("FEEDS") or {} - settings["FEEDS"] = { - build_url(file_path): feed for file_path, feed in FEEDS.items() - } - content: defaultdict[str, list[bytes]] = defaultdict(list) - spider_cls.start_urls = [self.mockserver.url("/")] - crawler = get_crawler(spider_cls, settings) - await crawler.crawl_async() - - for path, feed in FEEDS.items(): - dir_name = Path(path).parent - if not dir_name.exists(): - content[feed["format"]] = [] - continue - for file in sorted(dir_name.iterdir()): - content[feed["format"]].append(file.read_bytes()) - return content - - async def assertExportedJsonLines(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "jl" / self._file_mark: { - "format": "jl" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = await self.exported_data(items, settings) - for batch in data["jl"]: - got_batch = [ - json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() - ] - expected_batch, rows = rows[:batch_size], rows[batch_size:] - assert got_batch == expected_batch - - async def assertExportedCsv(self, items, header, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "csv" / self._file_mark: { - "format": "csv" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - data = await self.exported_data(items, settings) - for batch in data["csv"]: - got_batch = csv.DictReader(to_unicode(batch).splitlines()) - assert list(header) == got_batch.fieldnames - expected_batch, rows = rows[:batch_size], rows[batch_size:] - assert list(got_batch) == expected_batch - - async def assertExportedXml(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "xml" / self._file_mark: { - "format": "xml" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = await self.exported_data(items, settings) - for batch in data["xml"]: - root = lxml.etree.fromstring(batch) - got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] - expected_batch, rows = rows[:batch_size], rows[batch_size:] - assert got_batch == expected_batch - - async def assertExportedMultiple(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "xml" / self._file_mark: { - "format": "xml" - }, - self._random_temp_filename() / "json" / self._file_mark: { - "format": "json" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = await self.exported_data(items, settings) - # XML - xml_rows = rows.copy() - for batch in data["xml"]: - root = lxml.etree.fromstring(batch) - got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] - expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] - assert got_batch == expected_batch - # JSON - json_rows = rows.copy() - for batch in data["json"]: - got_batch = json.loads(batch.decode("utf-8")) - expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] - assert got_batch == expected_batch - - async def assertExportedPickle(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "pickle" / self._file_mark: { - "format": "pickle" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = await self.exported_data(items, settings) - - for batch in data["pickle"]: - got_batch = self._load_until_eof(batch, load_func=pickle.load) - expected_batch, rows = rows[:batch_size], rows[batch_size:] - assert got_batch == expected_batch - - async def assertExportedMarshal(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "marshal" / self._file_mark: { - "format": "marshal" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = await self.exported_data(items, settings) - - for batch in data["marshal"]: - got_batch = self._load_until_eof(batch, load_func=marshal.load) - expected_batch, rows = rows[:batch_size], rows[batch_size:] - assert got_batch == expected_batch - - @coroutine_test - async def test_export_items(self): - """Test partial deliveries in all supported formats""" - items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), - ] - rows = [ - {"egg": "spam1", "foo": "bar1", "baz": ""}, - {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, - {"foo": "bar3", "baz": "quux3", "egg": ""}, - ] - settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} - header = self.MyItem.fields.keys() - await self.assertExported(items, header, rows, settings=settings) - - def test_wrong_path(self): - """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" - settings = { - "FEEDS": { - self._random_temp_filename(): {"format": "xml"}, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - crawler = get_crawler(settings_dict=settings) - with pytest.raises(NotConfigured): - FeedExporter(crawler) - - @coroutine_test - async def test_export_no_items_not_store_empty(self): - for fmt in ("json", "jsonlines", "xml", "csv"): - settings = { - "FEEDS": { - self._random_temp_filename() / fmt / self._file_mark: { - "format": fmt - }, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - "FEED_STORE_EMPTY": False, - } - data = await self.exported_no_data(settings) - data = dict(data) - assert len(data[fmt]) == 0 - - @coroutine_test - async def test_export_no_items_store_empty(self): - formats = ( - ("json", b"[]"), - ("jsonlines", b""), - ("xml", b'\n'), - ("csv", b""), - ) - - for fmt, expctd in formats: - settings = { - "FEEDS": { - self._random_temp_filename() / fmt / self._file_mark: { - "format": fmt - }, - }, - "FEED_STORE_EMPTY": True, - "FEED_EXPORT_INDENT": None, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - data = await self.exported_no_data(settings) - data = dict(data) - assert data[fmt][0] == expctd - - @coroutine_test - async def test_export_multiple_configs(self): - items = [ - {"foo": "FOO", "bar": "BAR"}, - {"foo": "FOO1", "bar": "BAR1"}, - ] - - formats = { - "json": [ - b'[\n{"bar": "BAR"}\n]', - b'[\n{"bar": "BAR1"}\n]', - ], - "xml": [ - ( - b'\n' - b"\n \n FOO\n \n" - ), - ( - b'\n' - b"\n \n FOO1\n \n" - ), - ], - "csv": [ - b"foo,bar\r\nFOO,BAR\r\n", - b"foo,bar\r\nFOO1,BAR1\r\n", - ], - } - - settings = { - "FEEDS": { - self._random_temp_filename() / "json" / self._file_mark: { - "format": "json", - "indent": 0, - "fields": ["bar"], - "encoding": "utf-8", - }, - self._random_temp_filename() / "xml" / self._file_mark: { - "format": "xml", - "indent": 2, - "fields": ["foo"], - "encoding": "latin-1", - }, - self._random_temp_filename() / "csv" / self._file_mark: { - "format": "csv", - "indent": None, - "fields": ["foo", "bar"], - "encoding": "utf-8", - }, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - data = await self.exported_data(items, settings) - for fmt, expected in formats.items(): - for expected_batch, got_batch in zip(expected, data[fmt], strict=False): - assert got_batch == expected_batch - - @coroutine_test - async def test_batch_item_count_feeds_setting(self): - items = [{"foo": "FOO"}, {"foo": "FOO1"}] - formats = { - "json": [ - b'[{"foo": "FOO"}]', - b'[{"foo": "FOO1"}]', - ], - } - settings = { - "FEEDS": { - self._random_temp_filename() / "json" / self._file_mark: { - "format": "json", - "indent": None, - "encoding": "utf-8", - "batch_item_count": 1, - }, - }, - } - data = await self.exported_data(items, settings) - for fmt, expected in formats.items(): - for expected_batch, got_batch in zip(expected, data[fmt], strict=False): - assert got_batch == expected_batch - - @coroutine_test - async def test_batch_path_differ(self): - """ - Test that the name of all batch files differ from each other. - So %(batch_id)d replaced with the current id. - """ - items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), - ] - settings = { - "FEEDS": { - self._random_temp_filename() / "%(batch_id)d": { - "format": "json", - }, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - data = await self.exported_data(items, settings) - assert len(items) == len(data["json"]) - - @inline_callbacks_test - def test_stats_batch_file_success(self): - settings = { - "FEEDS": { - build_url( - str(self._random_temp_filename() / "json" / self._file_mark) - ): { - "format": "json", - } - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - crawler = get_crawler(ItemSpider, settings) - yield crawler.crawl(total=2, mockserver=self.mockserver) - assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() - assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 - - @pytest.mark.requires_boto3 - @inline_callbacks_test - def test_s3_export(self): - bucket = "mybucket" - items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), - ] - - class CustomS3FeedStorage(S3FeedStorage): - stubs = [] - - def open(self, *args, **kwargs): - from botocore import __version__ as botocore_version # noqa: PLC0415 - from botocore.stub import ANY, Stubber # noqa: PLC0415 - - expected_params = { - "Body": ANY, - "Bucket": bucket, - "Key": ANY, - } - if Version(botocore_version) >= Version("1.36.0"): - expected_params["ChecksumAlgorithm"] = ANY - - stub = Stubber(self.s3_client) - stub.activate() - CustomS3FeedStorage.stubs.append(stub) - stub.add_response( - "put_object", - expected_params=expected_params, - service_response={}, - ) - return super().open(*args, **kwargs) - - key = "export.csv" - uri = f"s3://{bucket}/{key}/%(batch_id)d.json" - batch_item_count = 1 - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "FEED_EXPORT_BATCH_ITEM_COUNT": batch_item_count, - "FEED_STORAGES": { - "s3": CustomS3FeedStorage, - }, - "FEEDS": { - uri: { - "format": "json", - }, - }, - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, uri) - verifyObject(IFeedStorage, storage) - - class TestSpider(scrapy.Spider): - name = "testspider" - - def parse(self, response): - yield from items - - TestSpider.start_urls = [self.mockserver.url("/")] - crawler = get_crawler(TestSpider, settings) - yield crawler.crawl() - - assert len(CustomS3FeedStorage.stubs) == len(items) - for stub in CustomS3FeedStorage.stubs[:-1]: - stub.assert_no_pending_responses() - assert ( - "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats() - ) - assert ( - crawler.stats.get_value("feedexport/success_count/CustomS3FeedStorage") == 3 - ) - - # Test that the FeedExporer sends the feed_exporter_closed and feed_slot_closed signals class TestFeedExporterSignals: items = [ @@ -2893,134 +1401,3 @@ class TestFeedExportInit: crawler = get_crawler(settings_dict=settings) exporter = FeedExporter.from_crawler(crawler) assert isinstance(exporter, FeedExporter) - - -class TestURIParams(ABC): - spider_name = "uri_params_spider" - deprecated_options = False - - @abstractmethod - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - raise NotImplementedError - - def _crawler_feed_exporter(self, settings): - if self.deprecated_options: - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings) - else: - crawler = get_crawler(settings_dict=settings) - feed_exporter = crawler.get_extension(FeedExporter) - return crawler, feed_exporter - - def test_default(self): - settings = self.build_settings( - uri="file:///tmp/%(name)s", - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - feed_exporter.open_spider(spider) - - assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" - - def test_none(self): - def uri_params(params, spider): - pass - - settings = self.build_settings( - uri="file:///tmp/%(name)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - - feed_exporter.open_spider(spider) - - assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" - - def test_empty_dict(self): - def uri_params(params, spider): - return {} - - settings = self.build_settings( - uri="file:///tmp/%(name)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - with pytest.raises(KeyError): - feed_exporter.open_spider(spider) - - def test_params_as_is(self): - def uri_params(params, spider): - return params - - settings = self.build_settings( - uri="file:///tmp/%(name)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - feed_exporter.open_spider(spider) - - assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" - - def test_custom_param(self): - def uri_params(params, spider): - return {**params, "foo": self.spider_name} - - settings = self.build_settings( - uri="file:///tmp/%(foo)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - feed_exporter.open_spider(spider) - - assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" - - -class TestURIParamsSetting(TestURIParams): - deprecated_options = True - - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - extra_settings = {} - if uri_params: - extra_settings["FEED_URI_PARAMS"] = uri_params - return { - "FEED_URI": uri, - **extra_settings, - } - - -class TestURIParamsFeedOption(TestURIParams): - deprecated_options = False - - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - options = { - "format": "jl", - } - if uri_params: - options["uri_params"] = uri_params - return { - "FEEDS": { - uri: options, - }, - } diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py new file mode 100644 index 000000000..3b70e896c --- /dev/null +++ b/tests/test_feedexport_batch.py @@ -0,0 +1,457 @@ +from __future__ import annotations + +import csv +import json +import marshal +import pickle +from collections import defaultdict +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urljoin + +import lxml.etree +import pytest +from packaging.version import Version +from zope.interface.verify import verifyObject + +import scrapy +from scrapy import Spider +from scrapy.exceptions import NotConfigured +from scrapy.extensions.feedexport import FeedExporter, IFeedStorage, S3FeedStorage +from scrapy.settings import Settings +from scrapy.utils.python import to_unicode +from scrapy.utils.test import get_crawler +from tests.spiders import ItemSpider +from tests.test_feedexport import TestFeedExportBase +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + from os import PathLike + + +def build_url(path: str | PathLike) -> str: + path_str = str(path) + if path_str[0] != "/": + path_str = "/" + path_str + return urljoin("file:", path_str) + + +class TestBatchDeliveries(TestFeedExportBase): + _file_mark = "_%(batch_time)s_#%(batch_id)02d_" + + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, list[bytes]]: + """Run spider with specified settings; return exported data.""" + + FEEDS = settings.get("FEEDS") or {} + settings["FEEDS"] = { + build_url(file_path): feed for file_path, feed in FEEDS.items() + } + content: defaultdict[str, list[bytes]] = defaultdict(list) + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + await crawler.crawl_async() + + for path, feed in FEEDS.items(): + dir_name = Path(path).parent + if not dir_name.exists(): + content[feed["format"]] = [] + continue + for file in sorted(dir_name.iterdir()): + content[feed["format"]].append(file.read_bytes()) + return content + + async def assertExportedJsonLines(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "jl" / self._file_mark: { + "format": "jl" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + for batch in data["jl"]: + got_batch = [ + json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() + ] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedCsv(self, items, header, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "csv" / self._file_mark: { + "format": "csv" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + data = await self.exported_data(items, settings) + for batch in data["csv"]: + got_batch = csv.DictReader(to_unicode(batch).splitlines()) + assert list(header) == got_batch.fieldnames + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert list(got_batch) == expected_batch + + async def assertExportedXml(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + for batch in data["xml"]: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedMultiple(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml" + }, + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + # XML + xml_rows = rows.copy() + for batch in data["xml"]: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] + expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] + assert got_batch == expected_batch + # JSON + json_rows = rows.copy() + for batch in data["json"]: + got_batch = json.loads(batch.decode("utf-8")) + expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedPickle(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "pickle" / self._file_mark: { + "format": "pickle" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + + for batch in data["pickle"]: + got_batch = self._load_until_eof(batch, load_func=pickle.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedMarshal(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "marshal" / self._file_mark: { + "format": "marshal" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + + for batch in data["marshal"]: + got_batch = self._load_until_eof(batch, load_func=marshal.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + @coroutine_test + async def test_export_items(self): + """Test partial deliveries in all supported formats""" + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), + ] + rows = [ + {"egg": "spam1", "foo": "bar1", "baz": ""}, + {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, + {"foo": "bar3", "baz": "quux3", "egg": ""}, + ] + settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} + header = self.MyItem.fields.keys() + await self.assertExported(items, header, rows, settings=settings) + + def test_wrong_path(self): + """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" + settings = { + "FEEDS": { + self._random_temp_filename(): {"format": "xml"}, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + crawler = get_crawler(settings_dict=settings) + with pytest.raises(NotConfigured): + FeedExporter(crawler) + + @coroutine_test + async def test_export_no_items_not_store_empty(self): + for fmt in ("json", "jsonlines", "xml", "csv"): + settings = { + "FEEDS": { + self._random_temp_filename() / fmt / self._file_mark: { + "format": fmt + }, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + "FEED_STORE_EMPTY": False, + } + data = await self.exported_no_data(settings) + data = dict(data) + assert len(data[fmt]) == 0 + + @coroutine_test + async def test_export_no_items_store_empty(self): + formats = ( + ("json", b"[]"), + ("jsonlines", b""), + ("xml", b'\n'), + ("csv", b""), + ) + + for fmt, expctd in formats: + settings = { + "FEEDS": { + self._random_temp_filename() / fmt / self._file_mark: { + "format": fmt + }, + }, + "FEED_STORE_EMPTY": True, + "FEED_EXPORT_INDENT": None, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + data = await self.exported_no_data(settings) + data = dict(data) + assert data[fmt][0] == expctd + + @coroutine_test + async def test_export_multiple_configs(self): + items = [ + {"foo": "FOO", "bar": "BAR"}, + {"foo": "FOO1", "bar": "BAR1"}, + ] + + formats = { + "json": [ + b'[\n{"bar": "BAR"}\n]', + b'[\n{"bar": "BAR1"}\n]', + ], + "xml": [ + ( + b'\n' + b"\n \n FOO\n \n" + ), + ( + b'\n' + b"\n \n FOO1\n \n" + ), + ], + "csv": [ + b"foo,bar\r\nFOO,BAR\r\n", + b"foo,bar\r\nFOO1,BAR1\r\n", + ], + } + + settings = { + "FEEDS": { + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json", + "indent": 0, + "fields": ["bar"], + "encoding": "utf-8", + }, + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml", + "indent": 2, + "fields": ["foo"], + "encoding": "latin-1", + }, + self._random_temp_filename() / "csv" / self._file_mark: { + "format": "csv", + "indent": None, + "fields": ["foo", "bar"], + "encoding": "utf-8", + }, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + data = await self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt], strict=False): + assert got_batch == expected_batch + + @coroutine_test + async def test_batch_item_count_feeds_setting(self): + items = [{"foo": "FOO"}, {"foo": "FOO1"}] + formats = { + "json": [ + b'[{"foo": "FOO"}]', + b'[{"foo": "FOO1"}]', + ], + } + settings = { + "FEEDS": { + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json", + "indent": None, + "encoding": "utf-8", + "batch_item_count": 1, + }, + }, + } + data = await self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt], strict=False): + assert got_batch == expected_batch + + @coroutine_test + async def test_batch_path_differ(self): + """ + Test that the name of all batch files differ from each other. + So %(batch_id)d replaced with the current id. + """ + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), + ] + settings = { + "FEEDS": { + self._random_temp_filename() / "%(batch_id)d": { + "format": "json", + }, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + data = await self.exported_data(items, settings) + assert len(items) == len(data["json"]) + + @inline_callbacks_test + def test_stats_batch_file_success(self): + settings = { + "FEEDS": { + build_url( + str(self._random_temp_filename() / "json" / self._file_mark) + ): { + "format": "json", + } + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + crawler = get_crawler(ItemSpider, settings) + yield crawler.crawl(total=2, mockserver=self.mockserver) + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 + + @pytest.mark.requires_boto3 + @inline_callbacks_test + def test_s3_export(self): + bucket = "mybucket" + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), + ] + + class CustomS3FeedStorage(S3FeedStorage): + stubs = [] + + def open(self, *args, **kwargs): + from botocore import __version__ as botocore_version # noqa: PLC0415 + from botocore.stub import ANY, Stubber # noqa: PLC0415 + + expected_params = { + "Body": ANY, + "Bucket": bucket, + "Key": ANY, + } + if Version(botocore_version) >= Version("1.36.0"): + expected_params["ChecksumAlgorithm"] = ANY + + stub = Stubber(self.s3_client) + stub.activate() + CustomS3FeedStorage.stubs.append(stub) + stub.add_response( + "put_object", + expected_params=expected_params, + service_response={}, + ) + return super().open(*args, **kwargs) + + key = "export.csv" + uri = f"s3://{bucket}/{key}/%(batch_id)d.json" + batch_item_count = 1 + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "FEED_EXPORT_BATCH_ITEM_COUNT": batch_item_count, + "FEED_STORAGES": { + "s3": CustomS3FeedStorage, + }, + "FEEDS": { + uri: { + "format": "json", + }, + }, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, uri) + verifyObject(IFeedStorage, storage) + + class TestSpider(scrapy.Spider): + name = "testspider" + + def parse(self, response): + yield from items + + TestSpider.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(TestSpider, settings) + yield crawler.crawl() + + assert len(CustomS3FeedStorage.stubs) == len(items) + for stub in CustomS3FeedStorage.stubs[:-1]: + stub.assert_no_pending_responses() + assert ( + "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats() + ) + assert ( + crawler.stats.get_value("feedexport/success_count/CustomS3FeedStorage") == 3 + ) diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py new file mode 100644 index 000000000..fa1c0586a --- /dev/null +++ b/tests/test_feedexport_postprocess.py @@ -0,0 +1,538 @@ +from __future__ import annotations + +import bz2 +import gzip +import lzma +import marshal +import pickle +import sys +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy.utils.test import get_crawler +from tests.test_feedexport import TestFeedExportBase, path_to_url, printf_escape +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy import Spider + + +class TestFeedPostProcessedExports(TestFeedExportBase): + items = [{"foo": "bar"}] + expected = b"foo\r\nbar\r\n" + + class MyPlugin1: + def __init__(self, file, feed_options): + self.file = file + self.feed_options = feed_options + self.char = self.feed_options.get("plugin1_char", b"") + + def write(self, data): + written_count = self.file.write(data) + written_count += self.file.write(self.char) + return written_count + + def close(self): + self.file.close() + + def _named_tempfile(self, name) -> str: + return str(Path(self.temp_dir, name)) + + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, bytes | None]: + """Run spider with specified settings; return exported data with filename.""" + + FEEDS = settings.get("FEEDS") or {} + settings["FEEDS"] = { + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() + } + + content: dict[str, bytes | None] = {} + try: + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + await crawler.crawl_async() + + for file_path in FEEDS: + content[str(file_path)] = ( + Path(file_path).read_bytes() if Path(file_path).exists() else None + ) + + finally: + for file_path in FEEDS: + if not Path(file_path).exists(): + continue + + Path(file_path).unlink() + + return content + + def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""): + data_stream = BytesIO() + gzipf = gzip.GzipFile( + fileobj=data_stream, + filename=filename, + mtime=mtime, + compresslevel=compresslevel, + mode="wb", + ) + gzipf.write(data) + gzipf.close() + data_stream.seek(0) + return data_stream.read() + + @coroutine_test + async def test_gzip_plugin(self): + filename = self._named_tempfile("gzip_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + }, + }, + } + + data = await self.exported_data(self.items, settings) + try: + gzip.decompress(data[filename]) + except OSError: + pytest.fail("Received invalid gzip data.") + + @coroutine_test + async def test_gzip_plugin_compresslevel(self): + filename_to_compressed = { + self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( + self.expected, compresslevel=0 + ), + self._named_tempfile("compresslevel_9"): self.get_gzip_compressed( + self.expected, compresslevel=9 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("compresslevel_0"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_compresslevel": 0, + "gzip_mtime": 0, + "gzip_filename": "", + }, + self._named_tempfile("compresslevel_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_compresslevel": 9, + "gzip_mtime": 0, + "gzip_filename": "", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_gzip_plugin_mtime(self): + filename_to_compressed = { + self._named_tempfile("mtime_123"): self.get_gzip_compressed( + self.expected, mtime=123 + ), + self._named_tempfile("mtime_123456789"): self.get_gzip_compressed( + self.expected, mtime=123456789 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("mtime_123"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 123, + "gzip_filename": "", + }, + self._named_tempfile("mtime_123456789"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 123456789, + "gzip_filename": "", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_gzip_plugin_filename(self): + filename_to_compressed = { + self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( + self.expected, filename="FILE1" + ), + self._named_tempfile("filename_FILE2"): self.get_gzip_compressed( + self.expected, filename="FILE2" + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("filename_FILE1"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 0, + "gzip_filename": "FILE1", + }, + self._named_tempfile("filename_FILE2"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 0, + "gzip_filename": "FILE2", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin(self): + filename = self._named_tempfile("lzma_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + }, + }, + } + + data = await self.exported_data(self.items, settings) + try: + lzma.decompress(data[filename]) + except lzma.LZMAError: + pytest.fail("Received invalid lzma data.") + + @coroutine_test + async def test_lzma_plugin_format(self): + filename_to_compressed = { + self._named_tempfile("format_FORMAT_XZ"): lzma.compress( + self.expected, format=lzma.FORMAT_XZ + ), + self._named_tempfile("format_FORMAT_ALONE"): lzma.compress( + self.expected, format=lzma.FORMAT_ALONE + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("format_FORMAT_XZ"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_format": lzma.FORMAT_XZ, + }, + self._named_tempfile("format_FORMAT_ALONE"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_format": lzma.FORMAT_ALONE, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin_check(self): + filename_to_compressed = { + self._named_tempfile("check_CHECK_NONE"): lzma.compress( + self.expected, check=lzma.CHECK_NONE + ), + self._named_tempfile("check_CHECK_CRC256"): lzma.compress( + self.expected, check=lzma.CHECK_SHA256 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("check_CHECK_NONE"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_check": lzma.CHECK_NONE, + }, + self._named_tempfile("check_CHECK_CRC256"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_check": lzma.CHECK_SHA256, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin_preset(self): + filename_to_compressed = { + self._named_tempfile("preset_PRESET_0"): lzma.compress( + self.expected, preset=0 + ), + self._named_tempfile("preset_PRESET_9"): lzma.compress( + self.expected, preset=9 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("preset_PRESET_0"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_preset": 0, + }, + self._named_tempfile("preset_PRESET_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_preset": 9, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin_filters(self): + if "PyPy" in sys.version: + # https://foss.heptapod.net/pypy/pypy/-/issues/3527 + pytest.skip("lzma filters doesn't work in PyPy") + + filters = [{"id": lzma.FILTER_LZMA2}] + compressed = lzma.compress(self.expected, filters=filters) + filename = self._named_tempfile("filters") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_filters": filters, + }, + }, + } + + data = await self.exported_data(self.items, settings) + assert compressed == data[filename] + result = lzma.decompress(data[filename]) + assert result == self.expected + + @coroutine_test + async def test_bz2_plugin(self): + filename = self._named_tempfile("bz2_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + }, + }, + } + + data = await self.exported_data(self.items, settings) + try: + bz2.decompress(data[filename]) + except OSError: + pytest.fail("Received invalid bz2 data.") + + @coroutine_test + async def test_bz2_plugin_compresslevel(self): + filename_to_compressed = { + self._named_tempfile("compresslevel_1"): bz2.compress( + self.expected, compresslevel=1 + ), + self._named_tempfile("compresslevel_9"): bz2.compress( + self.expected, compresslevel=9 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("compresslevel_1"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + "bz2_compresslevel": 1, + }, + self._named_tempfile("compresslevel_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + "bz2_compresslevel": 9, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = bz2.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_custom_plugin(self): + filename = self._named_tempfile("csv_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": [self.MyPlugin1], + }, + }, + } + + data = await self.exported_data(self.items, settings) + assert data[filename] == self.expected + + @coroutine_test + async def test_custom_plugin_with_parameter(self): + expected = b"foo\r\n\nbar\r\n\n" + filename = self._named_tempfile("newline") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": [self.MyPlugin1], + "plugin1_char": b"\n", + }, + }, + } + + data = await self.exported_data(self.items, settings) + assert data[filename] == expected + + @coroutine_test + async def test_custom_plugin_with_compression(self): + expected = b"foo\r\n\nbar\r\n\n" + + filename_to_decompressor = { + self._named_tempfile("bz2"): bz2.decompress, + self._named_tempfile("lzma"): lzma.decompress, + self._named_tempfile("gzip"): gzip.decompress, + } + + settings = { + "FEEDS": { + self._named_tempfile("bz2"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.Bz2Plugin", + ], + "plugin1_char": b"\n", + }, + self._named_tempfile("lzma"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.LZMAPlugin", + ], + "plugin1_char": b"\n", + }, + self._named_tempfile("gzip"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.GzipPlugin", + ], + "plugin1_char": b"\n", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, decompressor in filename_to_decompressor.items(): + result = decompressor(data[filename]) + assert result == expected + + @coroutine_test + async def test_exports_compatibility_with_postproc(self): + filename_to_expected = { + self._named_tempfile("csv"): b"foo\r\nbar\r\n", + self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]', + self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n', + self._named_tempfile("xml"): b'\n' + b"\nbar\n", + } + + settings = { + "FEEDS": { + self._named_tempfile("csv"): { + "format": "csv", + "postprocessing": [self.MyPlugin1], + # empty plugin to activate postprocessing.PostProcessingManager + }, + self._named_tempfile("json"): { + "format": "json", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("jsonlines"): { + "format": "jsonlines", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("xml"): { + "format": "xml", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("marshal"): { + "format": "marshal", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("pickle"): { + "format": "pickle", + "postprocessing": [self.MyPlugin1], + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, result in data.items(): + if "pickle" in filename: + expected, result = self.items[0], pickle.loads(result) + elif "marshal" in filename: + expected, result = self.items[0], marshal.loads(result) + else: + expected = filename_to_expected[filename] + assert result == expected diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py new file mode 100644 index 000000000..b4da9bc68 --- /dev/null +++ b/tests/test_feedexport_storages.py @@ -0,0 +1,558 @@ +from __future__ import annotations + +import os +import string +import tempfile +from io import BytesIO +from pathlib import Path +from typing import IO, Any +from unittest import mock +from urllib.parse import quote + +import pytest +from testfixtures import LogCapture +from w3lib.url import path_to_file_uri +from zope.interface.verify import verifyObject + +import scrapy +from scrapy.extensions.feedexport import ( + BlockingFeedStorage, + FileFeedStorage, + FTPFeedStorage, + GCSFeedStorage, + IFeedStorage, + S3FeedStorage, + StdoutFeedStorage, +) +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.test import get_crawler +from tests.mockserver.ftp import MockFTPServer +from tests.utils.decorators import coroutine_test + + +def mock_google_cloud_storage() -> tuple[Any, Any, Any]: + """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob + classes and set their proper return values. + """ + from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415 + + client_mock = mock.create_autospec(Client) + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + return (client_mock, bucket_mock, blob_mock) + + +class TestFileFeedStorage: + def test_store_file_uri(self, tmp_path): + path = tmp_path / "file.txt" + uri = path_to_file_uri(str(path)) + self._assert_stores(FileFeedStorage(uri), path) + + def test_store_file_uri_makedirs(self, tmp_path): + path = tmp_path / "more" / "paths" / "file.txt" + uri = path_to_file_uri(str(path)) + self._assert_stores(FileFeedStorage(uri), path) + + def test_store_direct_path(self, tmp_path): + path = tmp_path / "file.txt" + self._assert_stores(FileFeedStorage(str(path)), path) + + def test_store_direct_path_relative(self, tmp_path): + old_cwd = Path.cwd() + try: + os.chdir(tmp_path) + path = Path("foo", "bar") + self._assert_stores(FileFeedStorage(str(path)), path) + finally: + os.chdir(old_cwd) + + def test_interface(self, tmp_path): + path = tmp_path / "file.txt" + st = FileFeedStorage(str(path)) + verifyObject(IFeedStorage, st) + + @staticmethod + def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None: + storage = FileFeedStorage(str(path), feed_options=feed_options) + spider = scrapy.Spider("default") + file = storage.open(spider) + file.write(b"content") + storage.store(file) + + def test_append(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path) + self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") + + def test_overwrite(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path, {"overwrite": True}) + self._assert_stores( + FileFeedStorage(str(path), feed_options={"overwrite": True}), path + ) + + @staticmethod + def _assert_stores( + storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" + ) -> None: + spider = scrapy.Spider("default") + file = storage.open(spider) + file.write(b"content") + storage.store(file) + assert path.exists() + try: + assert path.read_bytes() == expected_content + finally: + path.unlink() + + def test_preserves_windows_path_without_file_scheme(self): + path = r"C:\Users\user\Desktop\test.txt" + storage = FileFeedStorage(path) + assert storage.path == path + + +@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage +class TestFTPFeedStorage: + def get_test_spider(self, settings=None): + class TestSpider(scrapy.Spider): + name = "test_spider" + + crawler = get_crawler(settings_dict=settings) + return TestSpider.from_crawler(crawler) + + async def _store(self, uri, content, feed_options=None, settings=None): + crawler = get_crawler(settings_dict=settings or {}) + storage = FTPFeedStorage.from_crawler( + crawler, + uri, + feed_options=feed_options, + ) + verifyObject(IFeedStorage, storage) + spider = self.get_test_spider() + file = storage.open(spider) + file.write(content) + await maybe_deferred_to_future(storage.store(file)) + + def _assert_stored(self, path: Path, content): + assert path.exists() + try: + assert path.read_bytes() == content + finally: + path.unlink() + + @coroutine_test + async def test_append(self): + with MockFTPServer() as ftp_server: + filename = "file" + url = ftp_server.url(filename) + feed_options = {"overwrite": False} + await self._store(url, b"foo", feed_options=feed_options) + await self._store(url, b"bar", feed_options=feed_options) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @coroutine_test + async def test_overwrite(self): + with MockFTPServer() as ftp_server: + filename = "file" + url = ftp_server.url(filename) + await self._store(url, b"foo") + await self._store(url, b"bar") + self._assert_stored(ftp_server.path / filename, b"bar") + + @coroutine_test + async def test_append_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {"FEED_STORAGE_FTP_ACTIVE": True} + filename = "file" + url = ftp_server.url(filename) + feed_options = {"overwrite": False} + await self._store(url, b"foo", feed_options=feed_options, settings=settings) + await self._store(url, b"bar", feed_options=feed_options, settings=settings) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @coroutine_test + async def test_overwrite_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {"FEED_STORAGE_FTP_ACTIVE": True} + filename = "file" + url = ftp_server.url(filename) + await self._store(url, b"foo", settings=settings) + await self._store(url, b"bar", settings=settings) + self._assert_stored(ftp_server.path / filename, b"bar") + + def test_uri_auth_quote(self): + # RFC3986: 3.2.1. User Information + pw_quoted = quote(string.punctuation, safe="") + st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) + assert st.password == string.punctuation + + +class MyBlockingFeedStorage(BlockingFeedStorage): + def _store_in_thread(self, file: IO[bytes]) -> None: + return + + +class TestBlockingFeedStorage: + def get_test_spider(self, settings=None): + class TestSpider(scrapy.Spider): + name = "test_spider" + + crawler = get_crawler(settings_dict=settings) + return TestSpider.from_crawler(crawler) + + def test_default_temp_dir(self): + b = MyBlockingFeedStorage() + + storage_file = b.open(self.get_test_spider()) + storage_dir = Path(storage_file.name).parent + assert str(storage_dir) == tempfile.gettempdir() + + def test_temp_file(self, tmp_path): + b = MyBlockingFeedStorage() + + spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) + storage_file = b.open(spider) + storage_dir = Path(storage_file.name).parent + assert storage_dir == tmp_path + + def test_invalid_folder(self, tmp_path): + b = MyBlockingFeedStorage() + + invalid_path = tmp_path / "invalid_path" + spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) + + with pytest.raises(OSError, match="Not a Directory:"): + b.open(spider=spider) + + +@pytest.mark.requires_boto3 +@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage +class TestS3FeedStorage: + def test_parse_credentials(self): + aws_credentials = { + "AWS_ACCESS_KEY_ID": "settings_key", + "AWS_SECRET_ACCESS_KEY": "settings_secret", + "AWS_SESSION_TOKEN": "settings_token", + } + crawler = get_crawler(settings_dict=aws_credentials) + # Instantiate with crawler + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "settings_key" + assert storage.secret_key == "settings_secret" + assert storage.session_token == "settings_token" + # Instantiate directly + storage = S3FeedStorage( + "s3://mybucket/export.csv", + aws_credentials["AWS_ACCESS_KEY_ID"], + aws_credentials["AWS_SECRET_ACCESS_KEY"], + session_token=aws_credentials["AWS_SESSION_TOKEN"], + ) + assert storage.access_key == "settings_key" + assert storage.secret_key == "settings_secret" + assert storage.session_token == "settings_token" + # URI priority > settings priority + storage = S3FeedStorage( + "s3://uri_key:uri_secret@mybucket/export.csv", + aws_credentials["AWS_ACCESS_KEY_ID"], + aws_credentials["AWS_SECRET_ACCESS_KEY"], + ) + assert storage.access_key == "uri_key" + assert storage.secret_key == "uri_secret" + + @coroutine_test + async def test_store(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + bucket = "mybucket" + key = "export.csv" + storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}") + verifyObject(IFeedStorage, storage) + + file = mock.MagicMock() + + storage.s3_client = mock.MagicMock() + await maybe_deferred_to_future(storage.store(file)) + assert storage.s3_client.upload_fileobj.call_args == mock.call( + Bucket=bucket, Key=key, Fileobj=file + ) + + def test_init_without_acl(self): + storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None + + def test_init_with_acl(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" + + def test_init_with_endpoint_url(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + endpoint_url="https://example.com", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url == "https://example.com" + + def test_init_with_region_name(self): + region_name = "ap-east-1" + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + region_name=region_name, + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name + + def test_from_crawler_without_acl(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None + + def test_without_endpoint_url(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url is None + + def test_without_region_name(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.s3_client._client_config.region_name == "us-east-1" + + def test_from_crawler_with_acl(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "FEED_STORAGE_S3_ACL": "custom-acl", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" + + def test_from_crawler_with_endpoint_url(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_ENDPOINT_URL": "https://example.com", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url == "https://example.com" + + def test_from_crawler_with_region_name(self): + region_name = "ap-east-1" + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_REGION_NAME": region_name, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name + + @coroutine_test + async def test_store_without_acl(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None + + storage.s3_client = mock.MagicMock() + await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) + acl = ( + storage.s3_client.upload_fileobj.call_args[1] + .get("ExtraArgs", {}) + .get("ACL") + ) + assert acl is None + + @coroutine_test + async def test_store_with_acl(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" + + storage.s3_client = mock.MagicMock() + await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) + acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] + assert acl == "custom-acl" + + def test_overwrite_default(self): + with LogCapture() as log: + S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert "S3 does not support appending to files" not in str(log) + + def test_overwrite_false(self): + with LogCapture() as log: + S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + "custom-acl", + feed_options={"overwrite": False}, + ) + assert "S3 does not support appending to files" in str(log) + + +@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage +class TestGCSFeedStorage: + def test_parse_settings(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") + assert storage.project_id == "123" + assert storage.acl == "publicRead" + assert storage.bucket_name == "mybucket" + assert storage.blob_name == "export.csv" + + def test_parse_empty_acl(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") + assert storage.acl is None + + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") + assert storage.acl is None + + @coroutine_test + async def test_store(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + uri = "gs://mybucket/export.csv" + project_id = "myproject-123" + acl = "publicRead" + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + with mock.patch("google.cloud.storage.Client") as m: + m.return_value = client_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id, acl) + await maybe_deferred_to_future(storage.store(f)) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with("mybucket") + bucket_mock.blob.assert_called_once_with("export.csv") + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + + def test_overwrite_default(self): + with LogCapture() as log: + GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") + assert "GCS does not support appending to files" not in str(log) + + def test_overwrite_false(self): + with LogCapture() as log: + GCSFeedStorage( + "gs://mybucket/export.csv", + "myproject-123", + "custom-acl", + feed_options={"overwrite": False}, + ) + assert "GCS does not support appending to files" in str(log) + + +class TestStdoutFeedStorage: + def test_store(self): + out = BytesIO() + storage = StdoutFeedStorage("stdout:", _stdout=out) + file = storage.open(scrapy.Spider("default")) + file.write(b"content") + storage.store(file) + assert out.getvalue() == b"content" + + def test_overwrite_default(self): + with LogCapture() as log: + StdoutFeedStorage("stdout:") + assert ( + "Standard output (stdout) storage does not support overwriting" + not in str(log) + ) + + def test_overwrite_true(self): + with LogCapture() as log: + StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) + assert "Standard output (stdout) storage does not support overwriting" in str( + log + ) diff --git a/tests/test_feedexport_uri_params.py b/tests/test_feedexport_uri_params.py new file mode 100644 index 000000000..150d8449f --- /dev/null +++ b/tests/test_feedexport_uri_params.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import warnings +from abc import ABC, abstractmethod + +import pytest + +import scrapy +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.extensions.feedexport import FeedExporter +from scrapy.utils.test import get_crawler + + +class TestURIParams(ABC): + spider_name = "uri_params_spider" + deprecated_options = False + + @abstractmethod + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): + raise NotImplementedError + + def _crawler_feed_exporter(self, settings): + if self.deprecated_options: + with pytest.warns( + ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", + ): + crawler = get_crawler(settings_dict=settings) + else: + crawler = get_crawler(settings_dict=settings) + feed_exporter = crawler.get_extension(FeedExporter) + return crawler, feed_exporter + + def test_default(self): + settings = self.build_settings( + uri="file:///tmp/%(name)s", + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + def test_none(self): + def uri_params(params, spider): + pass + + settings = self.build_settings( + uri="file:///tmp/%(name)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + def test_empty_dict(self): + def uri_params(params, spider): + return {} + + settings = self.build_settings( + uri="file:///tmp/%(name)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + with pytest.raises(KeyError): + feed_exporter.open_spider(spider) + + def test_params_as_is(self): + def uri_params(params, spider): + return params + + settings = self.build_settings( + uri="file:///tmp/%(name)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + def test_custom_param(self): + def uri_params(params, spider): + return {**params, "foo": self.spider_name} + + settings = self.build_settings( + uri="file:///tmp/%(foo)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + +class TestURIParamsSetting(TestURIParams): + deprecated_options = True + + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): + extra_settings = {} + if uri_params: + extra_settings["FEED_URI_PARAMS"] = uri_params + return { + "FEED_URI": uri, + **extra_settings, + } + + +class TestURIParamsFeedOption(TestURIParams): + deprecated_options = False + + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): + options = { + "format": "jl", + } + if uri_params: + options["uri_params"] = uri_params + return { + "FEEDS": { + uri: options, + }, + } diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 93cbcd1eb..06b280235 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,24 +1,12 @@ -import json -import re import warnings import xmlrpc.client from typing import Any -from unittest import mock -from urllib.parse import parse_qs, unquote_to_bytes import pytest -from scrapy.http import ( - FormRequest, - Headers, - HtmlResponse, - JsonRequest, - Request, - XmlRpcRequest, -) +from scrapy.http import Headers, Request, XmlRpcRequest from scrapy.http.request import NO_CALLBACK -from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.python import to_bytes class TestRequest: @@ -495,1022 +483,6 @@ class TestRequest: ) -class TestFormRequest(TestRequest): - request_class = FormRequest - - def assertQueryEqual(self, first, second, msg=None): - first = to_unicode(first).split("&") - second = to_unicode(second).split("&") - assert sorted(first) == sorted(second), msg - - def test_empty_formdata(self): - r1 = self.request_class("http://www.example.com", formdata={}) - assert r1.body == b"" - - def test_formdata_overrides_querystring(self): - data = (("a", "one"), ("a", "two"), ("b", "2")) - url = self.request_class( - "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data - ).url.split("#", maxsplit=1)[0] - fs = _qs(self.request_class(url, method="GET", formdata=data)) - assert set(fs[b"a"]) == {b"one", b"two"} - assert fs[b"b"] == [b"2"] - assert fs.get(b"c") is None - - data = {"a": "1", "b": "2"} - fs = _qs( - self.request_class("http://www.example.com/", method="GET", formdata=data) - ) - assert fs[b"a"] == [b"1"] - assert fs[b"b"] == [b"2"] - - def test_default_encoding_bytes(self): - # using default encoding (utf-8) - data = {b"one": b"two", b"price": b"\xc2\xa3 100"} - r2 = self.request_class("http://www.example.com", formdata=data) - assert r2.method == "POST" - assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") - assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" - - def test_default_encoding_textual_data(self): - # using default encoding (utf-8) - data = {"µ one": "two", "price": "£ 100"} - r2 = self.request_class("http://www.example.com", formdata=data) - assert r2.method == "POST" - assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") - assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" - - def test_default_encoding_mixed_data(self): - # using default encoding (utf-8) - data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} - r2 = self.request_class("http://www.example.com", formdata=data) - assert r2.method == "POST" - assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") - assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" - - def test_custom_encoding_bytes(self): - data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} - r2 = self.request_class( - "http://www.example.com", formdata=data, encoding="latin1" - ) - assert r2.method == "POST" - assert r2.encoding == "latin1" - self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") - assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" - - def test_custom_encoding_textual_data(self): - data = {"price": "£ 100"} - r3 = self.request_class( - "http://www.example.com", formdata=data, encoding="latin1" - ) - assert r3.encoding == "latin1" - assert r3.body == b"price=%A3+100" - - def test_multi_key_values(self): - # using multiples values for a single key - data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} - r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual( - r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" - ) - - def test_from_response_post(self): - response = _buildresponse( - b"""
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - assert req.method == "POST" - assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req) - assert set(fs[b"test"]) == {b"val1", b"val2"} - assert set(fs[b"one"]) == {b"two", b"three"} - assert fs[b"test2"] == [b"xxx"] - assert fs[b"six"] == [b"seven"] - - def test_from_response_post_nonascii_bytes_utf8(self): - response = _buildresponse( - b"""
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - assert req.method == "POST" - assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) - assert set(fs["test £"]) == {"val1", "val2"} - assert set(fs["one"]) == {"two", "three"} - assert fs["test2"] == ["xxx µ"] - assert fs["six"] == ["seven"] - - def test_from_response_post_nonascii_bytes_latin1(self): - response = _buildresponse( - b"""
- - - -
""", - url="http://www.example.com/this/list.html", - encoding="latin1", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - assert req.method == "POST" - assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True, encoding="latin1") - assert set(fs["test £"]) == {"val1", "val2"} - assert set(fs["one"]) == {"two", "three"} - assert fs["test2"] == ["xxx µ"] - assert fs["six"] == ["seven"] - - def test_from_response_post_nonascii_unicode(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - assert req.method == "POST" - assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) - assert set(fs["test £"]) == {"val1", "val2"} - assert set(fs["one"]) == {"two", "three"} - assert fs["test2"] == ["xxx µ"] - assert fs["six"] == ["seven"] - - def test_from_response_duplicate_form_key(self): - response = _buildresponse("
", url="http://www.example.com") - req = self.request_class.from_response( - response=response, - method="GET", - formdata=(("foo", "bar"), ("foo", "baz")), - ) - assert urlparse_cached(req).hostname == "www.example.com" - assert urlparse_cached(req).query == "foo=bar&foo=baz" - - def test_from_response_override_duplicate_form_key(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response( - response, formdata=(("two", "2"), ("two", "4")) - ) - fs = _qs(req) - assert fs[b"one"] == [b"1"] - assert fs[b"two"] == [b"2", b"4"] - - def test_from_response_extra_headers(self): - response = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response( - response=response, - formdata={"one": ["two", "three"], "six": "seven"}, - headers={"Accept-Encoding": "gzip,deflate"}, - ) - assert req.method == "POST" - assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" - assert req.headers["Accept-Encoding"] == b"gzip,deflate" - - def test_from_response_get(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - r1 = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - assert r1.method == "GET" - assert urlparse_cached(r1).hostname == "www.example.com" - assert urlparse_cached(r1).path == "/this/get.php" - fs = _qs(r1) - assert set(fs[b"test"]) == {b"val1", b"val2"} - assert set(fs[b"one"]) == {b"two", b"three"} - assert fs[b"test2"] == [b"xxx"] - assert fs[b"six"] == [b"seven"] - - def test_from_response_override_params(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response(response, formdata={"two": "2"}) - fs = _qs(req) - assert fs[b"one"] == [b"1"] - assert fs[b"two"] == [b"2"] - - def test_from_response_drop_params(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response(response, formdata={"two": None}) - fs = _qs(req) - assert fs[b"one"] == [b"1"] - assert b"two" not in fs - - def test_from_response_override_method(self): - response = _buildresponse( - """ -
- """ - ) - request = FormRequest.from_response(response) - assert request.method == "GET" - request = FormRequest.from_response(response, method="POST") - assert request.method == "POST" - - def test_from_response_override_url(self): - response = _buildresponse( - """ -
- """ - ) - request = FormRequest.from_response(response) - assert request.url == "http://example.com/app" - request = FormRequest.from_response(response, url="http://foo.bar/absolute") - assert request.url == "http://foo.bar/absolute" - request = FormRequest.from_response(response, url="/relative") - assert request.url == "http://example.com/relative" - - def test_from_response_case_insensitive(self): - response = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response(response) - fs = _qs(req) - assert fs[b"clickable1"] == [b"clicked1"] - assert b"i1" not in fs, fs # xpath in _get_inputs() - assert b"clickable2" not in fs, fs # xpath in _get_clickable() - - def test_from_response_submit_first_clickable(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response(response, formdata={"two": "2"}) - fs = _qs(req) - assert fs[b"clickable1"] == [b"clicked1"] - assert b"clickable2" not in fs, fs - assert fs[b"one"] == [b"1"] - assert fs[b"two"] == [b"2"] - - def test_from_response_submit_not_first_clickable(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response( - response, formdata={"two": "2"}, clickdata={"name": "clickable2"} - ) - fs = _qs(req) - assert fs[b"clickable2"] == [b"clicked2"] - assert b"clickable1" not in fs, fs - assert fs[b"one"] == [b"1"] - assert fs[b"two"] == [b"2"] - - def test_from_response_dont_submit_image_as_input(self): - response = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response(response, dont_click=True) - fs = _qs(req) - assert fs == {b"i1": [b"i1v"]} - - def test_from_response_dont_submit_reset_as_input(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response(response, dont_click=True) - fs = _qs(req) - assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} - - def test_from_response_clickdata_does_not_ignore_image(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response(response) - fs = _qs(req) - assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} - - def test_from_response_multiple_clickdata(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response( - response, clickdata={"name": "clickable", "value": "clicked2"} - ) - fs = _qs(req) - assert fs[b"clickable"] == [b"clicked2"] - assert fs[b"one"] == [b"clicked1"] - assert fs[b"two"] == [b"clicked2"] - - def test_from_response_unicode_clickdata(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response( - response, clickdata={"name": "price in \u00a3"} - ) - fs = _qs(req, to_unicode=True) - assert fs["price in \u00a3"] - - def test_from_response_unicode_clickdata_latin1(self): - response = _buildresponse( - """
- - - - -
""", - encoding="latin1", - ) - req = self.request_class.from_response( - response, clickdata={"name": "price in \u00a5"} - ) - fs = _qs(req, to_unicode=True, encoding="latin1") - assert fs["price in \u00a5"] - - def test_from_response_multiple_forms_clickdata(self): - response = _buildresponse( - """
- - -
-
- - -
- """ - ) - req = self.request_class.from_response( - response, formname="form2", clickdata={"name": "clickable"} - ) - fs = _qs(req) - assert fs[b"clickable"] == [b"clicked2"] - assert fs[b"field2"] == [b"value2"] - assert b"field1" not in fs, fs - - def test_from_response_override_clickable(self): - response = _buildresponse( - """
""" - ) - req = self.request_class.from_response( - response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} - ) - fs = _qs(req) - assert fs[b"clickme"] == [b"two"] - - def test_from_response_dont_click(self): - response = _buildresponse( - """
- - - - -
""" - ) - r1 = self.request_class.from_response(response, dont_click=True) - fs = _qs(r1) - assert b"clickable1" not in fs, fs - assert b"clickable2" not in fs, fs - - def test_from_response_ambiguous_clickdata(self): - response = _buildresponse( - """ -
- - - - -
""" - ) - with pytest.raises( - ValueError, - match=r"Multiple elements found .* matching the criteria in clickdata", - ): - self.request_class.from_response(response, clickdata={"type": "submit"}) - - def test_from_response_non_matching_clickdata(self): - response = _buildresponse( - """
- -
""" - ) - with pytest.raises( - ValueError, match="No clickable element matching clickdata:" - ): - self.request_class.from_response( - response, clickdata={"nonexistent": "notme"} - ) - - def test_from_response_nr_index_clickdata(self): - response = _buildresponse( - """
- - -
- """ - ) - req = self.request_class.from_response(response, clickdata={"nr": 1}) - fs = _qs(req) - assert b"clickable2" in fs - assert b"clickable1" not in fs - - def test_from_response_invalid_nr_index_clickdata(self): - response = _buildresponse( - """
- -
- """ - ) - with pytest.raises( - ValueError, match="No clickable element matching clickdata:" - ): - self.request_class.from_response(response, clickdata={"nr": 1}) - - def test_from_response_errors_noform(self): - response = _buildresponse("""""") - with pytest.raises(ValueError, match="No
element found in"): - self.request_class.from_response(response) - - def test_from_response_invalid_html5(self): - response = _buildresponse( - """""" - """""" - """
""" - ) - req = self.request_class.from_response(response, formdata={"bar": "buz"}) - fs = _qs(req) - assert fs == {b"foo": [b"xxx"], b"bar": [b"buz"]} - - def test_from_response_errors_formnumber(self): - response = _buildresponse( - """
- - - -
""" - ) - with pytest.raises(IndexError): - self.request_class.from_response(response, formnumber=1) - - def test_from_response_noformname(self): - response = _buildresponse( - """
- - -
""" - ) - r1 = self.request_class.from_response(response, formdata={"two": "3"}) - assert r1.method == "POST" - assert r1.headers["Content-type"] == b"application/x-www-form-urlencoded" - fs = _qs(r1) - assert fs == {b"one": [b"1"], b"two": [b"3"]} - - def test_from_response_formname_exists(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response(response, formname="form2") - assert r1.method == "POST" - fs = _qs(r1) - assert fs == {b"four": [b"4"], b"three": [b"3"]} - - def test_from_response_formname_nonexistent(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - r1 = self.request_class.from_response(response, formname="form3") - assert r1.method == "POST" - fs = _qs(r1) - assert fs == {b"one": [b"1"]} - - def test_from_response_formname_errors_formnumber(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - with pytest.raises(IndexError): - self.request_class.from_response(response, formname="form3", formnumber=2) - - def test_from_response_formid_exists(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response(response, formid="form2") - assert r1.method == "POST" - fs = _qs(r1) - assert fs == {b"four": [b"4"], b"three": [b"3"]} - - def test_from_response_formname_nonexistent_fallback_formid(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response( - response, formname="form3", formid="form2" - ) - assert r1.method == "POST" - fs = _qs(r1) - assert fs == {b"four": [b"4"], b"three": [b"3"]} - - def test_from_response_formid_nonexistent(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - r1 = self.request_class.from_response(response, formid="form3") - assert r1.method == "POST" - fs = _qs(r1) - assert fs == {b"one": [b"1"]} - - def test_from_response_formid_errors_formnumber(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - with pytest.raises(IndexError): - self.request_class.from_response(response, formid="form3", formnumber=2) - - def test_from_response_select(self): - res = _buildresponse( - """
- - - - - - - - - - - - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} - - def test_from_response_checkbox(self): - res = _buildresponse( - """
- - - - - - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} - - def test_from_response_input_text(self): - res = _buildresponse( - """
- - - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert fs == {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]} - - def test_from_response_input_hidden(self): - res = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert fs == {b"i1": [b"i1v1"], b"i2": [b""]} - - def test_from_response_input_textarea(self): - res = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert fs == {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]} - - def test_from_response_descendants(self): - res = _buildresponse( - """
-
-
- - -
- - - - -
- -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert set(fs) == {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"} - - def test_from_response_xpath(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response( - response, formxpath="//form[@action='post.php']" - ) - fs = _qs(r1) - assert fs[b"one"] == [b"1"] - - r1 = self.request_class.from_response( - response, formxpath="//form/input[@name='four']" - ) - fs = _qs(r1) - assert fs[b"three"] == [b"3"] - - with pytest.raises(ValueError, match="No
element found with"): - self.request_class.from_response( - response, formxpath="//form/input[@name='abc']" - ) - - def test_from_response_unicode_xpath(self): - response = _buildresponse(b'
') - r = self.request_class.from_response( - response, formxpath="//form[@name='\u044a']" - ) - fs = _qs(r) - assert not fs - - xpath = "//form[@name='\u03b1']" - with pytest.raises(ValueError, match=re.escape(xpath)): - self.request_class.from_response(response, formxpath=xpath) - - def test_from_response_button_submit(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - assert req.method == "POST" - assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req) - assert fs[b"test1"] == [b"val1"] - assert fs[b"test2"] == [b"val2"] - assert fs[b"button1"] == [b"submit1"] - - def test_from_response_button_notype(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - assert req.method == "POST" - assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req) - assert fs[b"test1"] == [b"val1"] - assert fs[b"test2"] == [b"val2"] - assert fs[b"button1"] == [b"submit1"] - - def test_from_response_submit_novalue(self): - response = _buildresponse( - """
- - - Submit -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - assert req.method == "POST" - assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req) - assert fs[b"test1"] == [b"val1"] - assert fs[b"test2"] == [b"val2"] - assert fs[b"button1"] == [b""] - - def test_from_response_button_novalue(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - assert req.method == "POST" - assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req) - assert fs[b"test1"] == [b"val1"] - assert fs[b"test2"] == [b"val2"] - assert fs[b"button1"] == [b""] - - def test_html_base_form_action(self): - response = _buildresponse( - """ - - - - - -
-
- - - """, - url="http://a.com/", - ) - req = self.request_class.from_response(response) - assert req.url == "http://b.com/test_form" - - def test_spaces_in_action(self): - resp = _buildresponse('
') - req = self.request_class.from_response(resp) - assert req.url == "http://example.com/path" - - def test_from_response_css(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response( - response, formcss="form[action='post.php']" - ) - fs = _qs(r1) - assert fs[b"one"] == [b"1"] - - r1 = self.request_class.from_response(response, formcss="input[name='four']") - fs = _qs(r1) - assert fs[b"three"] == [b"3"] - - with pytest.raises(ValueError, match="No
element found with"): - self.request_class.from_response(response, formcss="input[name='abc']") - - def test_from_response_valid_form_methods(self): - form_methods = [ - [method, method] for method in self.request_class.valid_form_methods - ] - form_methods.append(["UNKNOWN", "GET"]) - - for method, expected in form_methods: - response = _buildresponse( - f'' - '' - "
" - ) - r = self.request_class.from_response(response) - assert r.method == expected - - def test_form_response_with_invalid_formdata_type_error(self): - """Test that a ValueError is raised for non-iterable and non-dict formdata input""" - response = _buildresponse( - """ -
- -
- """ - ) - with pytest.raises( - ValueError, match="formdata should be a dict or iterable of tuples" - ): - FormRequest.from_response(response, formdata=123) - - def test_form_response_with_custom_invalid_formdata_value_error(self): - """Test that a ValueError is raised for fault-inducing iterable formdata input""" - response = _buildresponse( - """ -
- -
- """ - ) - - with pytest.raises( - ValueError, match="formdata should be a dict or iterable of tuples" - ): - FormRequest.from_response(response, formdata=("a",)) - - def test_get_form_with_xpath_no_form_parent(self): - """Test that _get_from raised a ValueError when an XPath selects an element - not nested within a
and no parent is found""" - response = _buildresponse( - """ -
-

This paragraph is not inside a form.

-
- - -
- """ - ) - - with pytest.raises(ValueError, match="No
element found with"): - FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') - - -def _buildresponse(body, **kwargs): - kwargs.setdefault("body", body) - kwargs.setdefault("url", "http://example.com") - kwargs.setdefault("encoding", "utf-8") - return HtmlResponse(**kwargs) - - -def _qs(req, encoding="utf-8", to_unicode=False): - qs = req.body if req.method == "POST" else req.url.partition("?")[2] - uqs = unquote_to_bytes(qs) - if to_unicode: - uqs = uqs.decode(encoding) - return parse_qs(uqs, True) - - class TestXmlRpcRequest(TestRequest): request_class = XmlRpcRequest default_method = "POST" @@ -1539,184 +511,3 @@ class TestXmlRpcRequest(TestRequest): def test_latin1(self): self._test_request(params=("pas£",), encoding="latin1") - - -class TestJsonRequest(TestRequest): - request_class = JsonRequest - default_method = "GET" - default_headers = { - b"Content-Type": [b"application/json"], - b"Accept": [b"application/json, text/javascript, */*; q=0.01"], - } - - def test_data(self): - r1 = self.request_class(url="http://www.example.com/") - assert r1.body == b"" - - body = b"body" - r2 = self.request_class(url="http://www.example.com/", body=body) - assert r2.body == body - - data = { - "name": "value", - } - r3 = self.request_class(url="http://www.example.com/", data=data) - assert r3.body == to_bytes(json.dumps(data)) - - # empty data - r4 = self.request_class(url="http://www.example.com/", data=[]) - assert r4.body == to_bytes(json.dumps([])) - - def test_data_method(self): - # data is not passed - r1 = self.request_class(url="http://www.example.com/") - assert r1.method == "GET" - - body = b"body" - r2 = self.request_class(url="http://www.example.com/", body=body) - assert r2.method == "GET" - - data = { - "name": "value", - } - r3 = self.request_class(url="http://www.example.com/", data=data) - assert r3.method == "POST" - - # method passed explicitly - r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") - assert r4.method == "GET" - - r5 = self.request_class(url="http://www.example.com/", data=[]) - assert r5.method == "POST" - - def test_body_data(self): - """passing both body and data should result a warning""" - body = b"body" - data = { - "name": "value", - } - with warnings.catch_warnings(record=True) as _warnings: - r5 = self.request_class(url="http://www.example.com/", body=body, data=data) - assert r5.body == body - assert r5.method == "GET" - assert len(_warnings) == 1 - assert "data will be ignored" in str(_warnings[0].message) - - def test_empty_body_data(self): - """passing any body value and data should result a warning""" - data = { - "name": "value", - } - with warnings.catch_warnings(record=True) as _warnings: - r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) - assert r6.body == b"" - assert r6.method == "GET" - assert len(_warnings) == 1 - assert "data will be ignored" in str(_warnings[0].message) - - def test_body_none_data(self): - data = { - "name": "value", - } - with warnings.catch_warnings(record=True) as _warnings: - r7 = self.request_class(url="http://www.example.com/", body=None, data=data) - assert r7.body == to_bytes(json.dumps(data)) - assert r7.method == "POST" - assert len(_warnings) == 0 - - def test_body_data_none(self): - with warnings.catch_warnings(record=True) as _warnings: - r8 = self.request_class(url="http://www.example.com/", body=None, data=None) - assert r8.method == "GET" - assert len(_warnings) == 0 - - def test_dumps_sort_keys(self): - """Test that sort_keys=True is passed to json.dumps by default""" - data = { - "name": "value", - } - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - self.request_class(url="http://www.example.com/", data=data) - kwargs = mock_dumps.call_args[1] - assert kwargs["sort_keys"] is True - - def test_dumps_kwargs(self): - """Test that dumps_kwargs are passed to json.dumps""" - data = { - "name": "value", - } - dumps_kwargs = { - "ensure_ascii": True, - "allow_nan": True, - } - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - self.request_class( - url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs - ) - kwargs = mock_dumps.call_args[1] - assert kwargs["ensure_ascii"] is True - assert kwargs["allow_nan"] is True - - def test_replace_data(self): - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - r1 = self.request_class(url="http://www.example.com/", data=data1) - r2 = r1.replace(data=data2) - assert r2.body == to_bytes(json.dumps(data2)) - - def test_replace_sort_keys(self): - """Test that replace provides sort_keys=True to json.dumps""" - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - r1 = self.request_class(url="http://www.example.com/", data=data1) - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - r1.replace(data=data2) - kwargs = mock_dumps.call_args[1] - assert kwargs["sort_keys"] is True - - def test_replace_dumps_kwargs(self): - """Test that dumps_kwargs are provided to json.dumps when replace is called""" - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - dumps_kwargs = { - "ensure_ascii": True, - "allow_nan": True, - } - r1 = self.request_class( - url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs - ) - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - r1.replace(data=data2) - kwargs = mock_dumps.call_args[1] - assert kwargs["ensure_ascii"] is True - assert kwargs["allow_nan"] is True - - def test_replacement_both_body_and_data_warns(self): - """Test that we get a warning if both body and data are passed""" - body1 = None - body2 = b"body" - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) - - with warnings.catch_warnings(record=True) as _warnings: - r1.replace(data=data2, body=body2) - assert "Both body and data passed. data will be ignored" in str( - _warnings[0].message - ) diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py new file mode 100644 index 000000000..92f2f3fac --- /dev/null +++ b/tests/test_http_request_form.py @@ -0,0 +1,1027 @@ +from __future__ import annotations + +import re +from urllib.parse import parse_qs, unquote_to_bytes + +import pytest + +from scrapy.http import FormRequest, HtmlResponse +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_unicode +from tests.test_http_request import TestRequest + + +def _buildresponse(body, **kwargs): + kwargs.setdefault("body", body) + kwargs.setdefault("url", "http://example.com") + kwargs.setdefault("encoding", "utf-8") + return HtmlResponse(**kwargs) + + +def _qs(req, encoding="utf-8", to_unicode=False): + qs = req.body if req.method == "POST" else req.url.partition("?")[2] + uqs = unquote_to_bytes(qs) + if to_unicode: + uqs = uqs.decode(encoding) + return parse_qs(uqs, True) + + +class TestFormRequest(TestRequest): + request_class = FormRequest + + def assertQueryEqual(self, first, second, msg=None): + first = to_unicode(first).split("&") + second = to_unicode(second).split("&") + assert sorted(first) == sorted(second), msg + + def test_empty_formdata(self): + r1 = self.request_class("http://www.example.com", formdata={}) + assert r1.body == b"" + + def test_formdata_overrides_querystring(self): + data = (("a", "one"), ("a", "two"), ("b", "2")) + url = self.request_class( + "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data + ).url.split("#", maxsplit=1)[0] + fs = _qs(self.request_class(url, method="GET", formdata=data)) + assert set(fs[b"a"]) == {b"one", b"two"} + assert fs[b"b"] == [b"2"] + assert fs.get(b"c") is None + + data = {"a": "1", "b": "2"} + fs = _qs( + self.request_class("http://www.example.com/", method="GET", formdata=data) + ) + assert fs[b"a"] == [b"1"] + assert fs[b"b"] == [b"2"] + + def test_default_encoding_bytes(self): + # using default encoding (utf-8) + data = {b"one": b"two", b"price": b"\xc2\xa3 100"} + r2 = self.request_class("http://www.example.com", formdata=data) + assert r2.method == "POST" + assert r2.encoding == "utf-8" + self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_default_encoding_textual_data(self): + # using default encoding (utf-8) + data = {"µ one": "two", "price": "£ 100"} + r2 = self.request_class("http://www.example.com", formdata=data) + assert r2.method == "POST" + assert r2.encoding == "utf-8" + self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_default_encoding_mixed_data(self): + # using default encoding (utf-8) + data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} + r2 = self.request_class("http://www.example.com", formdata=data) + assert r2.method == "POST" + assert r2.encoding == "utf-8" + self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_custom_encoding_bytes(self): + data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} + r2 = self.request_class( + "http://www.example.com", formdata=data, encoding="latin1" + ) + assert r2.method == "POST" + assert r2.encoding == "latin1" + self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_custom_encoding_textual_data(self): + data = {"price": "£ 100"} + r3 = self.request_class( + "http://www.example.com", formdata=data, encoding="latin1" + ) + assert r3.encoding == "latin1" + assert r3.body == b"price=%A3+100" + + def test_multi_key_values(self): + # using multiples values for a single key + data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} + r3 = self.request_class("http://www.example.com", formdata=data) + self.assertQueryEqual( + r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" + ) + + def test_from_response_post(self): + response = _buildresponse( + b""" + + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] + + def test_from_response_post_nonascii_bytes_utf8(self): + response = _buildresponse( + b"""
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req, to_unicode=True) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] + + def test_from_response_post_nonascii_bytes_latin1(self): + response = _buildresponse( + b"""
+ + + +
""", + url="http://www.example.com/this/list.html", + encoding="latin1", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req, to_unicode=True, encoding="latin1") + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] + + def test_from_response_post_nonascii_unicode(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req, to_unicode=True) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] + + def test_from_response_duplicate_form_key(self): + response = _buildresponse("
", url="http://www.example.com") + req = self.request_class.from_response( + response=response, + method="GET", + formdata=(("foo", "bar"), ("foo", "baz")), + ) + assert urlparse_cached(req).hostname == "www.example.com" + assert urlparse_cached(req).query == "foo=bar&foo=baz" + + def test_from_response_override_duplicate_form_key(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response( + response, formdata=(("two", "2"), ("two", "4")) + ) + fs = _qs(req) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2", b"4"] + + def test_from_response_extra_headers(self): + response = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response( + response=response, + formdata={"one": ["two", "three"], "six": "seven"}, + headers={"Accept-Encoding": "gzip,deflate"}, + ) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.headers["Accept-Encoding"] == b"gzip,deflate" + + def test_from_response_get(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + r1 = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + assert r1.method == "GET" + assert urlparse_cached(r1).hostname == "www.example.com" + assert urlparse_cached(r1).path == "/this/get.php" + fs = _qs(r1) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] + + def test_from_response_override_params(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response(response, formdata={"two": "2"}) + fs = _qs(req) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] + + def test_from_response_drop_params(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response(response, formdata={"two": None}) + fs = _qs(req) + assert fs[b"one"] == [b"1"] + assert b"two" not in fs + + def test_from_response_override_method(self): + response = _buildresponse( + """ +
+ """ + ) + request = FormRequest.from_response(response) + assert request.method == "GET" + request = FormRequest.from_response(response, method="POST") + assert request.method == "POST" + + def test_from_response_override_url(self): + response = _buildresponse( + """ +
+ """ + ) + request = FormRequest.from_response(response) + assert request.url == "http://example.com/app" + request = FormRequest.from_response(response, url="http://foo.bar/absolute") + assert request.url == "http://foo.bar/absolute" + request = FormRequest.from_response(response, url="/relative") + assert request.url == "http://example.com/relative" + + def test_from_response_case_insensitive(self): + response = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response(response) + fs = _qs(req) + assert fs[b"clickable1"] == [b"clicked1"] + assert b"i1" not in fs, fs # xpath in _get_inputs() + assert b"clickable2" not in fs, fs # xpath in _get_clickable() + + def test_from_response_submit_first_clickable(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response(response, formdata={"two": "2"}) + fs = _qs(req) + assert fs[b"clickable1"] == [b"clicked1"] + assert b"clickable2" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] + + def test_from_response_submit_not_first_clickable(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response( + response, formdata={"two": "2"}, clickdata={"name": "clickable2"} + ) + fs = _qs(req) + assert fs[b"clickable2"] == [b"clicked2"] + assert b"clickable1" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] + + def test_from_response_dont_submit_image_as_input(self): + response = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response(response, dont_click=True) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"]} + + def test_from_response_dont_submit_reset_as_input(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response(response, dont_click=True) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response(response) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} + + def test_from_response_multiple_clickdata(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response( + response, clickdata={"name": "clickable", "value": "clicked2"} + ) + fs = _qs(req) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"one"] == [b"clicked1"] + assert fs[b"two"] == [b"clicked2"] + + def test_from_response_unicode_clickdata(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response( + response, clickdata={"name": "price in \u00a3"} + ) + fs = _qs(req, to_unicode=True) + assert fs["price in \u00a3"] + + def test_from_response_unicode_clickdata_latin1(self): + response = _buildresponse( + """
+ + + + +
""", + encoding="latin1", + ) + req = self.request_class.from_response( + response, clickdata={"name": "price in \u00a5"} + ) + fs = _qs(req, to_unicode=True, encoding="latin1") + assert fs["price in \u00a5"] + + def test_from_response_multiple_forms_clickdata(self): + response = _buildresponse( + """
+ + +
+
+ + +
+ """ + ) + req = self.request_class.from_response( + response, formname="form2", clickdata={"name": "clickable"} + ) + fs = _qs(req) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"field2"] == [b"value2"] + assert b"field1" not in fs, fs + + def test_from_response_override_clickable(self): + response = _buildresponse( + """
""" + ) + req = self.request_class.from_response( + response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} + ) + fs = _qs(req) + assert fs[b"clickme"] == [b"two"] + + def test_from_response_dont_click(self): + response = _buildresponse( + """
+ + + + +
""" + ) + r1 = self.request_class.from_response(response, dont_click=True) + fs = _qs(r1) + assert b"clickable1" not in fs, fs + assert b"clickable2" not in fs, fs + + def test_from_response_ambiguous_clickdata(self): + response = _buildresponse( + """ +
+ + + + +
""" + ) + with pytest.raises( + ValueError, + match=r"Multiple elements found .* matching the criteria in clickdata", + ): + self.request_class.from_response(response, clickdata={"type": "submit"}) + + def test_from_response_non_matching_clickdata(self): + response = _buildresponse( + """
+ +
""" + ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response( + response, clickdata={"nonexistent": "notme"} + ) + + def test_from_response_nr_index_clickdata(self): + response = _buildresponse( + """
+ + +
+ """ + ) + req = self.request_class.from_response(response, clickdata={"nr": 1}) + fs = _qs(req) + assert b"clickable2" in fs + assert b"clickable1" not in fs + + def test_from_response_invalid_nr_index_clickdata(self): + response = _buildresponse( + """
+ +
+ """ + ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response(response, clickdata={"nr": 1}) + + def test_from_response_errors_noform(self): + response = _buildresponse("""""") + with pytest.raises(ValueError, match="No
element found in"): + self.request_class.from_response(response) + + def test_from_response_invalid_html5(self): + response = _buildresponse( + """""" + """""" + """
""" + ) + req = self.request_class.from_response(response, formdata={"bar": "buz"}) + fs = _qs(req) + assert fs == {b"foo": [b"xxx"], b"bar": [b"buz"]} + + def test_from_response_errors_formnumber(self): + response = _buildresponse( + """
+ + + +
""" + ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formnumber=1) + + def test_from_response_noformname(self): + response = _buildresponse( + """
+ + +
""" + ) + r1 = self.request_class.from_response(response, formdata={"two": "3"}) + assert r1.method == "POST" + assert r1.headers["Content-type"] == b"application/x-www-form-urlencoded" + fs = _qs(r1) + assert fs == {b"one": [b"1"], b"two": [b"3"]} + + def test_from_response_formname_exists(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response(response, formname="form2") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"four": [b"4"], b"three": [b"3"]} + + def test_from_response_formname_nonexistent(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + r1 = self.request_class.from_response(response, formname="form3") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"one": [b"1"]} + + def test_from_response_formname_errors_formnumber(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formname="form3", formnumber=2) + + def test_from_response_formid_exists(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response(response, formid="form2") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"four": [b"4"], b"three": [b"3"]} + + def test_from_response_formname_nonexistent_fallback_formid(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response( + response, formname="form3", formid="form2" + ) + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"four": [b"4"], b"three": [b"3"]} + + def test_from_response_formid_nonexistent(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + r1 = self.request_class.from_response(response, formid="form3") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"one": [b"1"]} + + def test_from_response_formid_errors_formnumber(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formid="form3", formnumber=2) + + def test_from_response_select(self): + res = _buildresponse( + """
+ + + + + + + + + + + + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} + + def test_from_response_checkbox(self): + res = _buildresponse( + """
+ + + + + + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} + + def test_from_response_input_text(self): + res = _buildresponse( + """
+ + + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]} + + def test_from_response_input_hidden(self): + res = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""]} + + def test_from_response_input_textarea(self): + res = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]} + + def test_from_response_descendants(self): + res = _buildresponse( + """
+
+
+ + +
+ + + + +
+ +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert set(fs) == {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"} + + def test_from_response_xpath(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response( + response, formxpath="//form[@action='post.php']" + ) + fs = _qs(r1) + assert fs[b"one"] == [b"1"] + + r1 = self.request_class.from_response( + response, formxpath="//form/input[@name='four']" + ) + fs = _qs(r1) + assert fs[b"three"] == [b"3"] + + with pytest.raises(ValueError, match="No
element found with"): + self.request_class.from_response( + response, formxpath="//form/input[@name='abc']" + ) + + def test_from_response_unicode_xpath(self): + response = _buildresponse(b'
') + r = self.request_class.from_response( + response, formxpath="//form[@name='\u044a']" + ) + fs = _qs(r) + assert not fs + + xpath = "//form[@name='\u03b1']" + with pytest.raises(ValueError, match=re.escape(xpath)): + self.request_class.from_response(response, formxpath=xpath) + + def test_from_response_button_submit(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] + + def test_from_response_button_notype(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] + + def test_from_response_submit_novalue(self): + response = _buildresponse( + """
+ + + Submit +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] + + def test_from_response_button_novalue(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] + + def test_html_base_form_action(self): + response = _buildresponse( + """ + + + + + +
+
+ + + """, + url="http://a.com/", + ) + req = self.request_class.from_response(response) + assert req.url == "http://b.com/test_form" + + def test_spaces_in_action(self): + resp = _buildresponse('
') + req = self.request_class.from_response(resp) + assert req.url == "http://example.com/path" + + def test_from_response_css(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response( + response, formcss="form[action='post.php']" + ) + fs = _qs(r1) + assert fs[b"one"] == [b"1"] + + r1 = self.request_class.from_response(response, formcss="input[name='four']") + fs = _qs(r1) + assert fs[b"three"] == [b"3"] + + with pytest.raises(ValueError, match="No
element found with"): + self.request_class.from_response(response, formcss="input[name='abc']") + + def test_from_response_valid_form_methods(self): + form_methods = [ + [method, method] for method in self.request_class.valid_form_methods + ] + form_methods.append(["UNKNOWN", "GET"]) + + for method, expected in form_methods: + response = _buildresponse( + f'' + '' + "
" + ) + r = self.request_class.from_response(response) + assert r.method == expected + + def test_form_response_with_invalid_formdata_type_error(self): + """Test that a ValueError is raised for non-iterable and non-dict formdata input""" + response = _buildresponse( + """ +
+ +
+ """ + ) + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): + FormRequest.from_response(response, formdata=123) + + def test_form_response_with_custom_invalid_formdata_value_error(self): + """Test that a ValueError is raised for fault-inducing iterable formdata input""" + response = _buildresponse( + """ +
+ +
+ """ + ) + + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): + FormRequest.from_response(response, formdata=("a",)) + + def test_get_form_with_xpath_no_form_parent(self): + """Test that _get_from raised a ValueError when an XPath selects an element + not nested within a
and no parent is found""" + response = _buildresponse( + """ +
+

This paragraph is not inside a form.

+
+ + +
+ """ + ) + + with pytest.raises(ValueError, match="No
element found with"): + FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') diff --git a/tests/test_http_request_json.py b/tests/test_http_request_json.py new file mode 100644 index 000000000..fcfe78365 --- /dev/null +++ b/tests/test_http_request_json.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +import json +import warnings +from unittest import mock + +from scrapy.http import JsonRequest +from scrapy.utils.python import to_bytes +from tests.test_http_request import TestRequest + + +class TestJsonRequest(TestRequest): + request_class = JsonRequest + default_method = "GET" + default_headers = { + b"Content-Type": [b"application/json"], + b"Accept": [b"application/json, text/javascript, */*; q=0.01"], + } + + def test_data(self): + r1 = self.request_class(url="http://www.example.com/") + assert r1.body == b"" + + body = b"body" + r2 = self.request_class(url="http://www.example.com/", body=body) + assert r2.body == body + + data = { + "name": "value", + } + r3 = self.request_class(url="http://www.example.com/", data=data) + assert r3.body == to_bytes(json.dumps(data)) + + # empty data + r4 = self.request_class(url="http://www.example.com/", data=[]) + assert r4.body == to_bytes(json.dumps([])) + + def test_data_method(self): + # data is not passed + r1 = self.request_class(url="http://www.example.com/") + assert r1.method == "GET" + + body = b"body" + r2 = self.request_class(url="http://www.example.com/", body=body) + assert r2.method == "GET" + + data = { + "name": "value", + } + r3 = self.request_class(url="http://www.example.com/", data=data) + assert r3.method == "POST" + + # method passed explicitly + r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") + assert r4.method == "GET" + + r5 = self.request_class(url="http://www.example.com/", data=[]) + assert r5.method == "POST" + + def test_body_data(self): + """passing both body and data should result a warning""" + body = b"body" + data = { + "name": "value", + } + with warnings.catch_warnings(record=True) as _warnings: + r5 = self.request_class(url="http://www.example.com/", body=body, data=data) + assert r5.body == body + assert r5.method == "GET" + assert len(_warnings) == 1 + assert "data will be ignored" in str(_warnings[0].message) + + def test_empty_body_data(self): + """passing any body value and data should result a warning""" + data = { + "name": "value", + } + with warnings.catch_warnings(record=True) as _warnings: + r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) + assert r6.body == b"" + assert r6.method == "GET" + assert len(_warnings) == 1 + assert "data will be ignored" in str(_warnings[0].message) + + def test_body_none_data(self): + data = { + "name": "value", + } + with warnings.catch_warnings(record=True) as _warnings: + r7 = self.request_class(url="http://www.example.com/", body=None, data=data) + assert r7.body == to_bytes(json.dumps(data)) + assert r7.method == "POST" + assert len(_warnings) == 0 + + def test_body_data_none(self): + with warnings.catch_warnings(record=True) as _warnings: + r8 = self.request_class(url="http://www.example.com/", body=None, data=None) + assert r8.method == "GET" + assert len(_warnings) == 0 + + def test_dumps_sort_keys(self): + """Test that sort_keys=True is passed to json.dumps by default""" + data = { + "name": "value", + } + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + self.request_class(url="http://www.example.com/", data=data) + kwargs = mock_dumps.call_args[1] + assert kwargs["sort_keys"] is True + + def test_dumps_kwargs(self): + """Test that dumps_kwargs are passed to json.dumps""" + data = { + "name": "value", + } + dumps_kwargs = { + "ensure_ascii": True, + "allow_nan": True, + } + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + self.request_class( + url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs + ) + kwargs = mock_dumps.call_args[1] + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True + + def test_replace_data(self): + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + r2 = r1.replace(data=data2) + assert r2.body == to_bytes(json.dumps(data2)) + + def test_replace_sort_keys(self): + """Test that replace provides sort_keys=True to json.dumps""" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + assert kwargs["sort_keys"] is True + + def test_replace_dumps_kwargs(self): + """Test that dumps_kwargs are provided to json.dumps when replace is called""" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + dumps_kwargs = { + "ensure_ascii": True, + "allow_nan": True, + } + r1 = self.request_class( + url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs + ) + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True + + def test_replacement_both_body_and_data_warns(self): + """Test that we get a warning if both body and data are passed""" + body1 = None + body2 = b"body" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) + + with warnings.catch_warnings(record=True) as _warnings: + r1.replace(data=data2, body=body2) + assert "Both body and data passed. data will be ignored" in str( + _warnings[0].message + ) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 8acdf25ec..1becf1542 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,27 +1,15 @@ -import codecs -from unittest import mock - import pytest from packaging.version import Version as parse_version from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding from scrapy.exceptions import NotSupported -from scrapy.http import ( - Headers, - HtmlResponse, - Request, - Response, - TextResponse, - XmlResponse, -) +from scrapy.http import Headers, Request, Response from scrapy.link import Link -from scrapy.selector import Selector -from scrapy.utils.python import to_unicode from tests import get_testdata -class TestResponseBase: +class TestResponse: response_class = Response def test_init(self): @@ -344,676 +332,3 @@ class TestResponseBase: def _links_response_no_href(self): body = get_testdata("link_extractor", "linkextractor_no_href.html") return self.response_class("http://example.com/index", body=body) - - -class TestTextResponse(TestResponseBase): - response_class = TextResponse - - def test_replace(self): - super().test_replace() - r1 = self.response_class( - "http://www.example.com", body="hello", encoding="cp852" - ) - r2 = r1.replace(url="http://www.example.com/other") - r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") - - assert isinstance(r2, self.response_class) - assert r2.url == "http://www.example.com/other" - self._assert_response_encoding(r2, "cp852") - assert r3.url == "http://www.example.com/other" - assert r3._declared_encoding() == "latin1" - - def test_unicode_url(self): - # instantiate with unicode url without encoding (should set default encoding) - resp = self.response_class("http://www.example.com/") - self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) - - # make sure urls are converted to str - resp = self.response_class(url="http://www.example.com/", encoding="utf-8") - assert isinstance(resp.url, str) - - resp = self.response_class( - url="http://www.example.com/price/\xa3", encoding="utf-8" - ) - assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") - resp = self.response_class( - url="http://www.example.com/price/\xa3", encoding="latin-1" - ) - assert resp.url == "http://www.example.com/price/\xa3" - resp = self.response_class( - "http://www.example.com/price/\xa3", - headers={"Content-type": ["text/html; charset=utf-8"]}, - ) - assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") - resp = self.response_class( - "http://www.example.com/price/\xa3", - headers={"Content-type": ["text/html; charset=iso-8859-1"]}, - ) - assert resp.url == "http://www.example.com/price/\xa3" - - def test_unicode_body(self): - unicode_string = ( - "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " - "\u0442\u0435\u043a\u0441\u0442" - ) - with pytest.raises(TypeError): - self.response_class("http://www.example.com", body="unicode body") - - original_string = unicode_string.encode("cp1251") - r1 = self.response_class( - "http://www.example.com", body=original_string, encoding="cp1251" - ) - - # check response.text - assert isinstance(r1.text, str) - assert r1.text == unicode_string - - def test_encoding(self): - r1 = self.response_class( - "http://www.example.com", - body=b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=utf-8"]}, - ) - r2 = self.response_class( - "http://www.example.com", encoding="utf-8", body="\xa3" - ) - r3 = self.response_class( - "http://www.example.com", - body=b"\xa3", - headers={"Content-type": ["text/html; charset=iso-8859-1"]}, - ) - r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") - r5 = self.response_class( - "http://www.example.com", - body=b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=None"]}, - ) - r6 = self.response_class( - "http://www.example.com", - body=b"\xa8D", - headers={"Content-type": ["text/html; charset=gb2312"]}, - ) - r7 = self.response_class( - "http://www.example.com", - body=b"\xa8D", - headers={"Content-type": ["text/html; charset=gbk"]}, - ) - r8 = self.response_class( - "http://www.example.com", - body=codecs.BOM_UTF8 + b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=cp1251"]}, - ) - r9 = self.response_class( - "http://www.example.com", - body=b"\x80", - headers={ - "Content-type": [b"application/x-download; filename=\x80dummy.txt"] - }, - ) - - assert r1._headers_encoding() == "utf-8" - assert r2._headers_encoding() is None - assert r2._declared_encoding() == "utf-8" - self._assert_response_encoding(r2, "utf-8") - assert r3._headers_encoding() == "cp1252" - assert r3._declared_encoding() == "cp1252" - assert r4._headers_encoding() is None - assert r5._headers_encoding() is None - assert r8._headers_encoding() == "cp1251" - assert r9._headers_encoding() is None - assert r8._declared_encoding() == "utf-8" - assert r9._declared_encoding() is None - self._assert_response_encoding(r5, "utf-8") - self._assert_response_encoding(r8, "utf-8") - self._assert_response_encoding(r9, "cp1252") - assert r4._body_inferred_encoding() is not None - assert r4._body_inferred_encoding() != "ascii" - self._assert_response_values(r1, "utf-8", "\xa3") - self._assert_response_values(r2, "utf-8", "\xa3") - self._assert_response_values(r3, "iso-8859-1", "\xa3") - self._assert_response_values(r6, "gb18030", "\u2015") - self._assert_response_values(r7, "gb18030", "\u2015") - self._assert_response_values(r9, "cp1252", "€") - - # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - with pytest.raises(TypeError): - self.response_class("http://www.example.com", body="\xa3") - - def test_declared_encoding_invalid(self): - """Check that unknown declared encodings are ignored""" - r = self.response_class( - "http://www.example.com", - headers={"Content-type": ["text/html; charset=UNKNOWN"]}, - body=b"\xc2\xa3", - ) - assert r._declared_encoding() is None - self._assert_response_values(r, "utf-8", "\xa3") - - def test_utf16(self): - """Test utf-16 because UnicodeDammit is known to have problems with""" - r = self.response_class( - "http://www.example.com", - body=b"\xff\xfeh\x00i\x00", - encoding="utf-16", - ) - self._assert_response_values(r, "utf-16", "hi") - - def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): - r6 = self.response_class( - "http://www.example.com", - headers={"Content-type": ["text/html; charset=utf-8"]}, - body=b"\xef\xbb\xbfWORD\xe3\xab", - ) - assert r6.encoding == "utf-8" - assert r6.text in { - "WORD\ufffd\ufffd", # w3lib < 1.19.0 - "WORD\ufffd", # w3lib >= 1.19.0 - } - - def test_bom_is_removed_from_body(self): - # Inferring encoding from body also cache decoded body as sideeffect, - # this test tries to ensure that calling response.encoding and - # response.text in indistinct order doesn't affect final - # response.text in indistinct order doesn't affect final - # values for encoding and decoded body. - url = "http://example.com" - body = b"\xef\xbb\xbfWORD" - headers = {"Content-type": ["text/html; charset=utf-8"]} - - # Test response without content-type and BOM encoding - response = self.response_class(url, body=body) - assert response.encoding == "utf-8" - assert response.text == "WORD" - response = self.response_class(url, body=body) - assert response.text == "WORD" - assert response.encoding == "utf-8" - - # Body caching sideeffect isn't triggered when encoding is declared in - # content-type header but BOM still need to be removed from decoded - # body - response = self.response_class(url, headers=headers, body=body) - assert response.encoding == "utf-8" - assert response.text == "WORD" - response = self.response_class(url, headers=headers, body=body) - assert response.text == "WORD" - assert response.encoding == "utf-8" - - def test_replace_wrong_encoding(self): - """Test invalid chars are replaced properly""" - r = self.response_class( - "http://www.example.com", - encoding="utf-8", - body=b"PREFIX\xe3\xabSUFFIX", - ) - # XXX: Policy for replacing invalid chars may suffer minor variations - # but it should always contain the unicode replacement char ('\ufffd') - assert "\ufffd" in r.text, repr(r.text) - assert "PREFIX" in r.text, repr(r.text) - assert "SUFFIX" in r.text, repr(r.text) - - # Do not destroy html tags due to encoding bugs - r = self.response_class( - "http://example.com", - encoding="utf-8", - body=b"\xf0value", - ) - assert "value" in r.text, repr(r.text) - - # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse - # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') - # assert '\ufffd' in r.text, repr(r.text) - - def test_selector(self): - body = b"Some page" - response = self.response_class("http://www.example.com", body=body) - - assert isinstance(response.selector, Selector) - assert response.selector.type == "html" - assert response.selector is response.selector # property is cached - assert response.selector.response is response - - assert response.selector.xpath("//title/text()").getall() == ["Some page"] - assert response.selector.css("title::text").getall() == ["Some page"] - assert response.selector.re("Some (.*)") == ["page"] - - def test_selector_shortcuts(self): - body = b"Some page" - response = self.response_class("http://www.example.com", body=body) - - assert ( - response.xpath("//title/text()").getall() - == response.selector.xpath("//title/text()").getall() - ) - assert ( - response.css("title::text").getall() - == response.selector.css("title::text").getall() - ) - - def test_selector_shortcuts_kwargs(self): - body = b'Some page

A nice paragraph.

' - response = self.response_class("http://www.example.com", body=body) - - assert ( - response.xpath( - "normalize-space(//p[@class=$pclass])", pclass="content" - ).getall() - == response.xpath('normalize-space(//p[@class="content"])').getall() - ) - assert ( - response.xpath( - "//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", - pcount=1, - ).getall() - == response.xpath( - '//title[count(following::p[@class="content"])=1]/text()' - ).getall() - ) - - def test_urljoin_with_base_url(self): - """Test urljoin shortcut which also evaluates base-url through get_base_url().""" - body = b'' - joined = self.response_class("http://www.example.com", body=body).urljoin( - "/test" - ) - absolute = "https://example.net/test" - assert joined == absolute - - body = b'' - joined = self.response_class("http://www.example.com", body=body).urljoin( - "test" - ) - absolute = "http://www.example.com/test" - assert joined == absolute - - body = b'' - joined = self.response_class("http://www.example.com", body=body).urljoin( - "test" - ) - absolute = "http://www.example.com/elsewhere/test" - assert joined == absolute - - def test_follow_selector(self): - resp = self._links_response() - urls = [ - "http://example.com/sample2.html", - "http://example.com/sample3.html", - "http://example.com/sample3.html", - "http://example.com/sample3.html", - "http://example.com/sample3.html#foo", - "http://www.google.com/something", - "http://example.com/innertag.html", - ] - - # select elements - for sellist in [resp.css("a"), resp.xpath("//a")]: - for sel, url in zip(sellist, urls, strict=False): - self._assert_followed_url(sel, url, response=resp) - - # select elements - self._assert_followed_url( - Selector(text='').css("link")[0], - "http://example.com/foo", - response=resp, - ) - - # href attributes should work - for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]: - for sel, url in zip(sellist, urls, strict=False): - self._assert_followed_url(sel, url, response=resp) - - # non-a elements are not supported - with pytest.raises( - ValueError, match="Only and elements are supported" - ): - resp.follow(resp.css("div")[0]) - - def test_follow_selector_list(self): - resp = self._links_response() - with pytest.raises(ValueError, match="SelectorList"): - resp.follow(resp.css("a")) - - def test_follow_selector_invalid(self): - resp = self._links_response() - with pytest.raises(ValueError, match="Unsupported"): - resp.follow(resp.xpath("count(//div)")[0]) - - def test_follow_selector_attribute(self): - resp = self._links_response() - for src in resp.css("img::attr(src)"): - self._assert_followed_url(src, "http://example.com/sample2.jpg") - - def test_follow_selector_no_href(self): - resp = self.response_class( - url="http://example.com", - body=b"click me", - ) - with pytest.raises(ValueError, match="no href"): - resp.follow(resp.css("a")[0]) - - def test_follow_whitespace_selector(self): - resp = self.response_class( - "http://example.com", - body=b"""click me""", - ) - self._assert_followed_url( - resp.css("a")[0], "http://example.com/foo", response=resp - ) - self._assert_followed_url( - resp.css("a::attr(href)")[0], - "http://example.com/foo", - response=resp, - ) - - def test_follow_encoding(self): - resp1 = self.response_class( - "http://example.com", - encoding="utf8", - body='click me'.encode(), - ) - req = self._assert_followed_url( - resp1.css("a")[0], - "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", - response=resp1, - ) - assert req.encoding == "utf8" - - resp2 = self.response_class( - "http://example.com", - encoding="cp1251", - body='click me'.encode( - "cp1251" - ), - ) - req = self._assert_followed_url( - resp2.css("a")[0], - "http://example.com/foo?%EF%F0%E8%E2%E5%F2", - response=resp2, - ) - assert req.encoding == "cp1251" - - def test_follow_flags(self): - res = self.response_class("http://example.com/") - fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - assert fol.flags == ["cached", "allowed"] - - def test_follow_all_flags(self): - re = self.response_class("http://www.example.com/") - urls = [ - "http://www.example.com/", - "http://www.example.com/2", - "http://www.example.com/foo", - ] - fol = re.follow_all(urls, flags=["cached", "allowed"]) - for req in fol: - assert req.flags == ["cached", "allowed"] - - def test_follow_all_css(self): - expected = [ - "http://example.com/sample3.html", - "http://example.com/innertag.html", - ] - response = self._links_response() - extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] - assert expected == extracted - - def test_follow_all_css_skip_invalid(self): - expected = [ - "http://example.com/page/1/", - "http://example.com/page/3/", - "http://example.com/page/4/", - ] - response = self._links_response_no_href() - extracted1 = [r.url for r in response.follow_all(css=".pagination a")] - assert expected == extracted1 - extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))] - assert expected == extracted2 - - def test_follow_all_xpath(self): - expected = [ - "http://example.com/sample3.html", - "http://example.com/innertag.html", - ] - response = self._links_response() - extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') - assert expected == [r.url for r in extracted] - - def test_follow_all_xpath_skip_invalid(self): - expected = [ - "http://example.com/page/1/", - "http://example.com/page/3/", - "http://example.com/page/4/", - ] - response = self._links_response_no_href() - extracted1 = [ - r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a') - ] - assert expected == extracted1 - extracted2 = [ - r.url - for r in response.follow_all(response.xpath('//div[@id="pagination"]/a')) - ] - assert expected == extracted2 - - def test_follow_all_too_many_arguments(self): - response = self._links_response() - with pytest.raises( - ValueError, match="Please supply exactly one of the following arguments" - ): - response.follow_all( - css='a[href*="example.com"]', - xpath='//a[contains(@href, "example.com")]', - ) - - def test_json_response(self): - json_body = b"""{"ip": "109.187.217.200"}""" - json_response = self.response_class("http://www.example.com", body=json_body) - assert json_response.json() == {"ip": "109.187.217.200"} - - text_body = b"""text""" - text_response = self.response_class("http://www.example.com", body=text_body) - with pytest.raises( - ValueError, match=r"(Expecting value|Unexpected '<'): line 1" - ): - text_response.json() - - def test_cache_json_response(self): - json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] - for json_body in json_valid_bodies: - json_response = self.response_class( - "http://www.example.com", body=json_body - ) - - with mock.patch("json.loads") as mock_json: - for _ in range(2): - json_response.json() - mock_json.assert_called_once_with(json_body) - - -class TestHtmlResponse(TestTextResponse): - response_class = HtmlResponse - - def test_html_encoding(self): - body = b"""Some page - - Price: \xa3100' - """ - r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, "iso-8859-1", body) - - body = b""" - - Price: \xa3100 - """ - r2 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r2, "iso-8859-1", body) - - # for conflicting declarations headers must take precedence - body = b"""Some page - - Price: \xa3100' - """ - r3 = self.response_class( - "http://www.example.com", - body=body, - headers={"Content-type": ["text/html; charset=iso-8859-1"]}, - ) - self._assert_response_values(r3, "iso-8859-1", body) - - # make sure replace() preserves the encoding of the original response - body = b"New body \xa3" - r4 = r3.replace(body=body) - self._assert_response_values(r4, "iso-8859-1", body) - - def test_html5_meta_charset(self): - body = b"""Some pagebla bla""" - r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, "gb2312", body) - - -class TestXmlResponse(TestTextResponse): - response_class = XmlResponse - - def test_xml_encoding(self): - body = b"" - r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body) - - body = b"""""" - r2 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r2, "iso-8859-1", body) - - # make sure replace() preserves the explicit encoding passed in the __init__ method - body = b"""""" - r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8") - body2 = b"New body" - r4 = r3.replace(body=body2) - self._assert_response_values(r4, "utf-8", body2) - - def test_replace_encoding(self): - # make sure replace() keeps the previous encoding unless overridden explicitly - body = b"""""" - body2 = b"""""" - r5 = self.response_class("http://www.example.com", body=body) - r6 = r5.replace(body=body2) - r7 = r5.replace(body=body2, encoding="utf-8") - self._assert_response_values(r5, "iso-8859-1", body) - self._assert_response_values(r6, "iso-8859-1", body2) - self._assert_response_values(r7, "utf-8", body2) - - def test_selector(self): - body = b'value' - response = self.response_class("http://www.example.com", body=body) - - assert isinstance(response.selector, Selector) - assert response.selector.type == "xml" - assert response.selector is response.selector # property is cached - assert response.selector.response is response - - assert response.selector.xpath("//elem/text()").getall() == ["value"] - - def test_selector_shortcuts(self): - body = b'value' - response = self.response_class("http://www.example.com", body=body) - - assert ( - response.xpath("//elem/text()").getall() - == response.selector.xpath("//elem/text()").getall() - ) - - def test_selector_shortcuts_kwargs(self): - body = b""" - - value - """ - response = self.response_class("http://www.example.com", body=body) - - assert ( - response.xpath( - "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall() - == response.selector.xpath( - "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall() - ) - - response.selector.register_namespace("s2", "http://scrapy.org") - assert ( - response.xpath( - "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"} - ).getall() - == response.selector.xpath("//s2:elem/text()").getall() - ) - - -class CustomResponse(TextResponse): - attributes = (*TextResponse.attributes, "foo", "bar") - - def __init__(self, *args, **kwargs) -> None: - self.foo = kwargs.pop("foo", None) - self.bar = kwargs.pop("bar", None) - self.lost = kwargs.pop("lost", None) - super().__init__(*args, **kwargs) - - -class TestCustomResponse(TestTextResponse): - response_class = CustomResponse - - def test_copy(self): - super().test_copy() - r1 = self.response_class( - url="https://example.org", - status=200, - foo="foo", - bar="bar", - lost="lost", - ) - r2 = r1.copy() - assert isinstance(r2, self.response_class) - assert r1.foo == r2.foo - assert r1.bar == r2.bar - assert r1.lost == "lost" - assert r2.lost is None - - def test_replace(self): - super().test_replace() - r1 = self.response_class( - url="https://example.org", - status=200, - foo="foo", - bar="bar", - lost="lost", - ) - - r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") - assert isinstance(r2, self.response_class) - assert r1.foo == "foo" - assert r1.bar == "bar" - assert r1.lost == "lost" - assert r2.foo == "new-foo" - assert r2.bar == "new-bar" - assert r2.lost == "new-lost" - - r3 = r1.replace(foo="new-foo", bar="new-bar") - assert isinstance(r3, self.response_class) - assert r1.foo == "foo" - assert r1.bar == "bar" - assert r1.lost == "lost" - assert r3.foo == "new-foo" - assert r3.bar == "new-bar" - assert r3.lost is None - - r4 = r1.replace(foo="new-foo") - assert isinstance(r4, self.response_class) - assert r1.foo == "foo" - assert r1.bar == "bar" - assert r1.lost == "lost" - assert r4.foo == "new-foo" - assert r4.bar == "bar" - assert r4.lost is None - - with pytest.raises( - TypeError, - match=r"__init__\(\) got an unexpected keyword argument 'unknown'", - ): - r1.replace(unknown="unknown") diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py new file mode 100644 index 000000000..c16af52b9 --- /dev/null +++ b/tests/test_http_response_text.py @@ -0,0 +1,684 @@ +from __future__ import annotations + +import codecs +from unittest import mock + +import pytest + +from scrapy.http import HtmlResponse, TextResponse, XmlResponse +from scrapy.selector import Selector +from scrapy.utils.python import to_unicode +from tests.test_http_response import TestResponse + + +class TestTextResponse(TestResponse): + response_class = TextResponse + + def test_replace(self): + super().test_replace() + r1 = self.response_class( + "http://www.example.com", body="hello", encoding="cp852" + ) + r2 = r1.replace(url="http://www.example.com/other") + r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") + + assert isinstance(r2, self.response_class) + assert r2.url == "http://www.example.com/other" + self._assert_response_encoding(r2, "cp852") + assert r3.url == "http://www.example.com/other" + assert r3._declared_encoding() == "latin1" + + def test_unicode_url(self): + # instantiate with unicode url without encoding (should set default encoding) + resp = self.response_class("http://www.example.com/") + self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) + + # make sure urls are converted to str + resp = self.response_class(url="http://www.example.com/", encoding="utf-8") + assert isinstance(resp.url, str) + + resp = self.response_class( + url="http://www.example.com/price/\xa3", encoding="utf-8" + ) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") + resp = self.response_class( + url="http://www.example.com/price/\xa3", encoding="latin-1" + ) + assert resp.url == "http://www.example.com/price/\xa3" + resp = self.response_class( + "http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}, + ) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") + resp = self.response_class( + "http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + assert resp.url == "http://www.example.com/price/\xa3" + + def test_unicode_body(self): + unicode_string = ( + "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " + "\u0442\u0435\u043a\u0441\u0442" + ) + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="unicode body") + + original_string = unicode_string.encode("cp1251") + r1 = self.response_class( + "http://www.example.com", body=original_string, encoding="cp1251" + ) + + # check response.text + assert isinstance(r1.text, str) + assert r1.text == unicode_string + + def test_encoding(self): + r1 = self.response_class( + "http://www.example.com", + body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}, + ) + r2 = self.response_class( + "http://www.example.com", encoding="utf-8", body="\xa3" + ) + r3 = self.response_class( + "http://www.example.com", + body=b"\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") + r5 = self.response_class( + "http://www.example.com", + body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=None"]}, + ) + r6 = self.response_class( + "http://www.example.com", + body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gb2312"]}, + ) + r7 = self.response_class( + "http://www.example.com", + body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gbk"]}, + ) + r8 = self.response_class( + "http://www.example.com", + body=codecs.BOM_UTF8 + b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=cp1251"]}, + ) + r9 = self.response_class( + "http://www.example.com", + body=b"\x80", + headers={ + "Content-type": [b"application/x-download; filename=\x80dummy.txt"] + }, + ) + + assert r1._headers_encoding() == "utf-8" + assert r2._headers_encoding() is None + assert r2._declared_encoding() == "utf-8" + self._assert_response_encoding(r2, "utf-8") + assert r3._headers_encoding() == "cp1252" + assert r3._declared_encoding() == "cp1252" + assert r4._headers_encoding() is None + assert r5._headers_encoding() is None + assert r8._headers_encoding() == "cp1251" + assert r9._headers_encoding() is None + assert r8._declared_encoding() == "utf-8" + assert r9._declared_encoding() is None + self._assert_response_encoding(r5, "utf-8") + self._assert_response_encoding(r8, "utf-8") + self._assert_response_encoding(r9, "cp1252") + assert r4._body_inferred_encoding() is not None + assert r4._body_inferred_encoding() != "ascii" + self._assert_response_values(r1, "utf-8", "\xa3") + self._assert_response_values(r2, "utf-8", "\xa3") + self._assert_response_values(r3, "iso-8859-1", "\xa3") + self._assert_response_values(r6, "gb18030", "\u2015") + self._assert_response_values(r7, "gb18030", "\u2015") + self._assert_response_values(r9, "cp1252", "€") + + # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="\xa3") + + def test_declared_encoding_invalid(self): + """Check that unknown declared encodings are ignored""" + r = self.response_class( + "http://www.example.com", + headers={"Content-type": ["text/html; charset=UNKNOWN"]}, + body=b"\xc2\xa3", + ) + assert r._declared_encoding() is None + self._assert_response_values(r, "utf-8", "\xa3") + + def test_utf16(self): + """Test utf-16 because UnicodeDammit is known to have problems with""" + r = self.response_class( + "http://www.example.com", + body=b"\xff\xfeh\x00i\x00", + encoding="utf-16", + ) + self._assert_response_values(r, "utf-16", "hi") + + def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): + r6 = self.response_class( + "http://www.example.com", + headers={"Content-type": ["text/html; charset=utf-8"]}, + body=b"\xef\xbb\xbfWORD\xe3\xab", + ) + assert r6.encoding == "utf-8" + assert r6.text in { + "WORD\ufffd\ufffd", # w3lib < 1.19.0 + "WORD\ufffd", # w3lib >= 1.19.0 + } + + def test_bom_is_removed_from_body(self): + # Inferring encoding from body also cache decoded body as sideeffect, + # this test tries to ensure that calling response.encoding and + # response.text in indistinct order doesn't affect final + # response.text in indistinct order doesn't affect final + # values for encoding and decoded body. + url = "http://example.com" + body = b"\xef\xbb\xbfWORD" + headers = {"Content-type": ["text/html; charset=utf-8"]} + + # Test response without content-type and BOM encoding + response = self.response_class(url, body=body) + assert response.encoding == "utf-8" + assert response.text == "WORD" + response = self.response_class(url, body=body) + assert response.text == "WORD" + assert response.encoding == "utf-8" + + # Body caching sideeffect isn't triggered when encoding is declared in + # content-type header but BOM still need to be removed from decoded + # body + response = self.response_class(url, headers=headers, body=body) + assert response.encoding == "utf-8" + assert response.text == "WORD" + response = self.response_class(url, headers=headers, body=body) + assert response.text == "WORD" + assert response.encoding == "utf-8" + + def test_replace_wrong_encoding(self): + """Test invalid chars are replaced properly""" + r = self.response_class( + "http://www.example.com", + encoding="utf-8", + body=b"PREFIX\xe3\xabSUFFIX", + ) + # XXX: Policy for replacing invalid chars may suffer minor variations + # but it should always contain the unicode replacement char ('\ufffd') + assert "\ufffd" in r.text, repr(r.text) + assert "PREFIX" in r.text, repr(r.text) + assert "SUFFIX" in r.text, repr(r.text) + + # Do not destroy html tags due to encoding bugs + r = self.response_class( + "http://example.com", + encoding="utf-8", + body=b"\xf0value", + ) + assert "value" in r.text, repr(r.text) + + # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse + # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') + # assert '\ufffd' in r.text, repr(r.text) + + def test_selector(self): + body = b"Some page" + response = self.response_class("http://www.example.com", body=body) + + assert isinstance(response.selector, Selector) + assert response.selector.type == "html" + assert response.selector is response.selector # property is cached + assert response.selector.response is response + + assert response.selector.xpath("//title/text()").getall() == ["Some page"] + assert response.selector.css("title::text").getall() == ["Some page"] + assert response.selector.re("Some (.*)") == ["page"] + + def test_selector_shortcuts(self): + body = b"Some page" + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath("//title/text()").getall() + == response.selector.xpath("//title/text()").getall() + ) + assert ( + response.css("title::text").getall() + == response.selector.css("title::text").getall() + ) + + def test_selector_shortcuts_kwargs(self): + body = b'Some page

A nice paragraph.

' + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath( + "normalize-space(//p[@class=$pclass])", pclass="content" + ).getall() + == response.xpath('normalize-space(//p[@class="content"])').getall() + ) + assert ( + response.xpath( + "//title[count(following::p[@class=$pclass])=$pcount]/text()", + pclass="content", + pcount=1, + ).getall() + == response.xpath( + '//title[count(following::p[@class="content"])=1]/text()' + ).getall() + ) + + def test_urljoin_with_base_url(self): + """Test urljoin shortcut which also evaluates base-url through get_base_url().""" + body = b'' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "/test" + ) + absolute = "https://example.net/test" + assert joined == absolute + + body = b'' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "test" + ) + absolute = "http://www.example.com/test" + assert joined == absolute + + body = b'' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "test" + ) + absolute = "http://www.example.com/elsewhere/test" + assert joined == absolute + + def test_follow_selector(self): + resp = self._links_response() + urls = [ + "http://example.com/sample2.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html#foo", + "http://www.google.com/something", + "http://example.com/innertag.html", + ] + + # select elements + for sellist in [resp.css("a"), resp.xpath("//a")]: + for sel, url in zip(sellist, urls, strict=False): + self._assert_followed_url(sel, url, response=resp) + + # select elements + self._assert_followed_url( + Selector(text='').css("link")[0], + "http://example.com/foo", + response=resp, + ) + + # href attributes should work + for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]: + for sel, url in zip(sellist, urls, strict=False): + self._assert_followed_url(sel, url, response=resp) + + # non-a elements are not supported + with pytest.raises( + ValueError, match="Only and elements are supported" + ): + resp.follow(resp.css("div")[0]) + + def test_follow_selector_list(self): + resp = self._links_response() + with pytest.raises(ValueError, match="SelectorList"): + resp.follow(resp.css("a")) + + def test_follow_selector_invalid(self): + resp = self._links_response() + with pytest.raises(ValueError, match="Unsupported"): + resp.follow(resp.xpath("count(//div)")[0]) + + def test_follow_selector_attribute(self): + resp = self._links_response() + for src in resp.css("img::attr(src)"): + self._assert_followed_url(src, "http://example.com/sample2.jpg") + + def test_follow_selector_no_href(self): + resp = self.response_class( + url="http://example.com", + body=b"click me", + ) + with pytest.raises(ValueError, match="no href"): + resp.follow(resp.css("a")[0]) + + def test_follow_whitespace_selector(self): + resp = self.response_class( + "http://example.com", + body=b"""click me""", + ) + self._assert_followed_url( + resp.css("a")[0], "http://example.com/foo", response=resp + ) + self._assert_followed_url( + resp.css("a::attr(href)")[0], + "http://example.com/foo", + response=resp, + ) + + def test_follow_encoding(self): + resp1 = self.response_class( + "http://example.com", + encoding="utf8", + body='click me'.encode(), + ) + req = self._assert_followed_url( + resp1.css("a")[0], + "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", + response=resp1, + ) + assert req.encoding == "utf8" + + resp2 = self.response_class( + "http://example.com", + encoding="cp1251", + body='click me'.encode( + "cp1251" + ), + ) + req = self._assert_followed_url( + resp2.css("a")[0], + "http://example.com/foo?%EF%F0%E8%E2%E5%F2", + response=resp2, + ) + assert req.encoding == "cp1251" + + def test_follow_flags(self): + res = self.response_class("http://example.com/") + fol = res.follow("http://example.com/", flags=["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] + + def test_follow_all_flags(self): + re = self.response_class("http://www.example.com/") + urls = [ + "http://www.example.com/", + "http://www.example.com/2", + "http://www.example.com/foo", + ] + fol = re.follow_all(urls, flags=["cached", "allowed"]) + for req in fol: + assert req.flags == ["cached", "allowed"] + + def test_follow_all_css(self): + expected = [ + "http://example.com/sample3.html", + "http://example.com/innertag.html", + ] + response = self._links_response() + extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] + assert expected == extracted + + def test_follow_all_css_skip_invalid(self): + expected = [ + "http://example.com/page/1/", + "http://example.com/page/3/", + "http://example.com/page/4/", + ] + response = self._links_response_no_href() + extracted1 = [r.url for r in response.follow_all(css=".pagination a")] + assert expected == extracted1 + extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))] + assert expected == extracted2 + + def test_follow_all_xpath(self): + expected = [ + "http://example.com/sample3.html", + "http://example.com/innertag.html", + ] + response = self._links_response() + extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') + assert expected == [r.url for r in extracted] + + def test_follow_all_xpath_skip_invalid(self): + expected = [ + "http://example.com/page/1/", + "http://example.com/page/3/", + "http://example.com/page/4/", + ] + response = self._links_response_no_href() + extracted1 = [ + r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a') + ] + assert expected == extracted1 + extracted2 = [ + r.url + for r in response.follow_all(response.xpath('//div[@id="pagination"]/a')) + ] + assert expected == extracted2 + + def test_follow_all_too_many_arguments(self): + response = self._links_response() + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): + response.follow_all( + css='a[href*="example.com"]', + xpath='//a[contains(@href, "example.com")]', + ) + + def test_json_response(self): + json_body = b"""{"ip": "109.187.217.200"}""" + json_response = self.response_class("http://www.example.com", body=json_body) + assert json_response.json() == {"ip": "109.187.217.200"} + + text_body = b"""text""" + text_response = self.response_class("http://www.example.com", body=text_body) + with pytest.raises( + ValueError, match=r"(Expecting value|Unexpected '<'): line 1" + ): + text_response.json() + + def test_cache_json_response(self): + json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] + for json_body in json_valid_bodies: + json_response = self.response_class( + "http://www.example.com", body=json_body + ) + + with mock.patch("json.loads") as mock_json: + for _ in range(2): + json_response.json() + mock_json.assert_called_once_with(json_body) + + +class TestHtmlResponse(TestTextResponse): + response_class = HtmlResponse + + def test_html_encoding(self): + body = b"""Some page + + Price: \xa3100' + """ + r1 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r1, "iso-8859-1", body) + + body = b""" + + Price: \xa3100 + """ + r2 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r2, "iso-8859-1", body) + + # for conflicting declarations headers must take precedence + body = b"""Some page + + Price: \xa3100' + """ + r3 = self.response_class( + "http://www.example.com", + body=body, + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + self._assert_response_values(r3, "iso-8859-1", body) + + # make sure replace() preserves the encoding of the original response + body = b"New body \xa3" + r4 = r3.replace(body=body) + self._assert_response_values(r4, "iso-8859-1", body) + + def test_html5_meta_charset(self): + body = b"""Some pagebla bla""" + r1 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r1, "gb2312", body) + + +class TestXmlResponse(TestTextResponse): + response_class = XmlResponse + + def test_xml_encoding(self): + body = b"" + r1 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body) + + body = b"""""" + r2 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r2, "iso-8859-1", body) + + # make sure replace() preserves the explicit encoding passed in the __init__ method + body = b"""""" + r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8") + body2 = b"New body" + r4 = r3.replace(body=body2) + self._assert_response_values(r4, "utf-8", body2) + + def test_replace_encoding(self): + # make sure replace() keeps the previous encoding unless overridden explicitly + body = b"""""" + body2 = b"""""" + r5 = self.response_class("http://www.example.com", body=body) + r6 = r5.replace(body=body2) + r7 = r5.replace(body=body2, encoding="utf-8") + self._assert_response_values(r5, "iso-8859-1", body) + self._assert_response_values(r6, "iso-8859-1", body2) + self._assert_response_values(r7, "utf-8", body2) + + def test_selector(self): + body = b'value' + response = self.response_class("http://www.example.com", body=body) + + assert isinstance(response.selector, Selector) + assert response.selector.type == "xml" + assert response.selector is response.selector # property is cached + assert response.selector.response is response + + assert response.selector.xpath("//elem/text()").getall() == ["value"] + + def test_selector_shortcuts(self): + body = b'value' + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath("//elem/text()").getall() + == response.selector.xpath("//elem/text()").getall() + ) + + def test_selector_shortcuts_kwargs(self): + body = b""" + + value + """ + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath( + "//s:elem/text()", namespaces={"s": "http://scrapy.org"} + ).getall() + == response.selector.xpath( + "//s:elem/text()", namespaces={"s": "http://scrapy.org"} + ).getall() + ) + + response.selector.register_namespace("s2", "http://scrapy.org") + assert ( + response.xpath( + "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"} + ).getall() + == response.selector.xpath("//s2:elem/text()").getall() + ) + + +class CustomResponse(TextResponse): + attributes = (*TextResponse.attributes, "foo", "bar") + + def __init__(self, *args, **kwargs) -> None: + self.foo = kwargs.pop("foo", None) + self.bar = kwargs.pop("bar", None) + self.lost = kwargs.pop("lost", None) + super().__init__(*args, **kwargs) + + +class TestCustomResponse(TestTextResponse): + response_class = CustomResponse + + def test_copy(self): + super().test_copy() + r1 = self.response_class( + url="https://example.org", + status=200, + foo="foo", + bar="bar", + lost="lost", + ) + r2 = r1.copy() + assert isinstance(r2, self.response_class) + assert r1.foo == r2.foo + assert r1.bar == r2.bar + assert r1.lost == "lost" + assert r2.lost is None + + def test_replace(self): + super().test_replace() + r1 = self.response_class( + url="https://example.org", + status=200, + foo="foo", + bar="bar", + lost="lost", + ) + + r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") + assert isinstance(r2, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r2.foo == "new-foo" + assert r2.bar == "new-bar" + assert r2.lost == "new-lost" + + r3 = r1.replace(foo="new-foo", bar="new-bar") + assert isinstance(r3, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r3.foo == "new-foo" + assert r3.bar == "new-bar" + assert r3.lost is None + + r4 = r1.replace(foo="new-foo") + assert isinstance(r4, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r4.foo == "new-foo" + assert r4.bar == "bar" + assert r4.lost is None + + with pytest.raises( + TypeError, + match=r"__init__\(\) got an unexpected keyword argument 'unknown'", + ): + r1.replace(unknown="unknown") diff --git a/tests/test_spider.py b/tests/test_spider.py index 68d4f85db..ff9aed74f 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,35 +1,19 @@ from __future__ import annotations -import gzip -import re -import warnings -from datetime import datetime -from io import BytesIO -from logging import ERROR, WARNING -from pathlib import Path from typing import Any from unittest import mock import pytest from testfixtures import LogCapture -from w3lib.url import safe_url_string from scrapy import signals from scrapy.crawler import Crawler -from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse -from scrapy.linkextractors import LinkExtractor +from scrapy.http import Response, TextResponse, XmlResponse from scrapy.settings import Settings -from scrapy.spiders import ( - CrawlSpider, - CSVFeedSpider, - Rule, - SitemapSpider, - Spider, - XMLFeedSpider, -) +from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider from scrapy.spiders.init import InitSpider from scrapy.utils.test import get_crawler, get_reactor_settings -from tests import get_testdata, tests_datadir +from tests import get_testdata from tests.utils.decorators import coroutine_test, inline_callbacks_test @@ -229,629 +213,6 @@ class TestCSVFeedSpider(TestSpider): assert len(rows) == 4 -class TestCrawlSpider(TestSpider): - test_body = b"""Page title - -

Item 12

-
-

About us

-
-
-

This shouldn't be followed

-
- """ - spider_class = CrawlSpider - - def test_rule_without_link_extractor(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(),) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ] - - def test_process_links(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) - - def dummy_process_links(self, links): - return links - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ] - - def test_process_links_filter(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_links="filter_process_links"),) - _test_regex = re.compile("nofollow") - - def filter_process_links(self, links): - return [link for link in links if not self._test_regex.search(link.url)] - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 2 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - ] - - def test_process_links_generator(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) - - def dummy_process_links(self, links): - yield from links - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ] - - def test_process_request(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - def process_request_change_domain(request, response): - return request.replace(url=request.url.replace(".org", ".com")) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = ( - Rule(LinkExtractor(), process_request=process_request_change_domain), - ) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.com/somepage/item/12.html", - "http://example.com/about.html", - "http://example.com/nofollow.html", - ] - - def test_process_request_with_response(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - def process_request_meta_response_class(request, response): - request.meta["response_class"] = response.__class__.__name__ - return request - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = ( - Rule( - LinkExtractor(), process_request=process_request_meta_response_class - ), - ) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ] - assert [r.meta["response_class"] for r in output] == [ - "HtmlResponse", - "HtmlResponse", - "HtmlResponse", - ] - - def test_process_request_instance_method(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_request="process_request_upper"),) - - def process_request_upper(self, request, response): - return request.replace(url=request.url.upper()) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), - safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), - safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), - ] - - def test_process_request_instance_method_with_response(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = ( - Rule( - LinkExtractor(), - process_request="process_request_meta_response_class", - ), - ) - - def process_request_meta_response_class(self, request, response): - request.meta["response_class"] = response.__class__.__name__ - return request - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ] - assert [r.meta["response_class"] for r in output] == [ - "HtmlResponse", - "HtmlResponse", - "HtmlResponse", - ] - - def test_follow_links_attribute_population(self): - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - assert hasattr(spider, "_follow_links") - assert spider._follow_links - - settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} - crawler = get_crawler(settings_dict=settings_dict) - spider = self.spider_class.from_crawler(crawler, "example.com") - assert hasattr(spider, "_follow_links") - assert not spider._follow_links - - @inline_callbacks_test - def test_start_url(self): - class TestSpider(self.spider_class): - name = "test" - start_url = "https://www.example.com" - - crawler = get_crawler(TestSpider) - with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: - yield crawler.crawl() - assert "Error while reading start items and requests" in str(log) - assert "did you miss an 's'?" in str(log) - - def test_parse_response_use(self): - class _CrawlSpider(CrawlSpider): - name = "test" - start_urls = "https://www.example.com" - _follow_links = False - - with warnings.catch_warnings(record=True) as w: - spider = _CrawlSpider() - assert len(w) == 0 - spider._parse_response( - TextResponse(spider.start_urls, body=b""), None, None - ) - assert len(w) == 1 - - def test_parse_response_override(self): - class _CrawlSpider(CrawlSpider): - def _parse_response(self, response, callback, cb_kwargs, follow=True): - pass - - name = "test" - start_urls = "https://www.example.com" - _follow_links = False - - with warnings.catch_warnings(record=True) as w: - assert len(w) == 0 - spider = _CrawlSpider() - assert len(w) == 1 - spider._parse_response( - TextResponse(spider.start_urls, body=b""), None, None - ) - assert len(w) == 1 - - def test_parse_with_rules(self): - class _CrawlSpider(CrawlSpider): - name = "test" - start_urls = "https://www.example.com" - - with warnings.catch_warnings(record=True) as w: - spider = _CrawlSpider() - spider.parse_with_rules( - TextResponse(spider.start_urls, body=b""), None, None - ) - assert len(w) == 0 - - -class TestSitemapSpider(TestSpider): - spider_class = SitemapSpider - - BODY = b"SITEMAP" - f = BytesIO() - g = gzip.GzipFile(fileobj=f, mode="w+b") - g.write(BODY) - g.close() - GZBODY = f.getvalue() - - def assertSitemapBody(self, response: Response, body: bytes | None) -> None: - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - assert spider._get_sitemap_body(response) == body - - def test_get_sitemap_body(self): - r = XmlResponse(url="http://www.example.com/", body=self.BODY) - self.assertSitemapBody(r, self.BODY) - - r = HtmlResponse(url="http://www.example.com/", body=self.BODY) - self.assertSitemapBody(r, None) - - r = Response(url="http://www.example.com/favicon.ico", body=self.BODY) - self.assertSitemapBody(r, None) - - def test_get_sitemap_body_gzip_headers(self): - r = Response( - url="http://www.example.com/sitemap", - body=self.GZBODY, - headers={"content-type": "application/gzip"}, - request=Request("http://www.example.com/sitemap"), - ) - self.assertSitemapBody(r, self.BODY) - - def test_get_sitemap_body_xml_url(self): - r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY) - self.assertSitemapBody(r, self.BODY) - - def test_get_sitemap_body_xml_url_compressed(self): - r = Response( - url="http://www.example.com/sitemap.xml.gz", - body=self.GZBODY, - request=Request("http://www.example.com/sitemap"), - ) - self.assertSitemapBody(r, self.BODY) - - # .xml.gz but body decoded by HttpCompression middleware already - r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) - self.assertSitemapBody(r, self.BODY) - - def test_get_sitemap_urls_from_robotstxt(self): - robots = b"""# Sitemap files -Sitemap: http://example.com/sitemap.xml -Sitemap: http://example.com/sitemap-product-index.xml -Sitemap: HTTP://example.com/sitemap-uppercase.xml -Sitemap: /sitemap-relative-url.xml -""" - - r = TextResponse(url="http://www.example.com/robots.txt", body=robots) - spider = self.spider_class("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://example.com/sitemap.xml", - "http://example.com/sitemap-product-index.xml", - "http://example.com/sitemap-uppercase.xml", - "http://www.example.com/sitemap-relative-url.xml", - ] - - def test_alternate_url_locs(self): - sitemap = b""" - - - http://www.example.com/english/ - - - - - - """ - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/english/" - ] - - spider.sitemap_alternate_links = True - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/english/", - "http://www.example.com/deutsch/", - "http://www.example.com/schweiz-deutsch/", - "http://www.example.com/italiano/", - ] - - def test_sitemap_filter(self): - sitemap = b""" - - - http://www.example.com/english/ - 2010-01-01 - - - http://www.example.com/portuguese/ - 2005-01-01 - - """ - - class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, entries): - for entry in entries: - date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") - if date_time.year > 2008: - yield entry - - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/english/", - "http://www.example.com/portuguese/", - ] - - spider = FilteredSitemapSpider("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/english/" - ] - - def test_sitemap_filter_with_alternate_links(self): - sitemap = b""" - - - http://www.example.com/english/article_1/ - 2010-01-01 - - - - http://www.example.com/english/article_2/ - 2015-01-01 - - """ - - class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, entries): - for entry in entries: - alternate_links = entry.get("alternate", ()) - for link in alternate_links: - if "/deutsch/" in link: - entry["loc"] = link - yield entry - - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/english/article_1/", - "http://www.example.com/english/article_2/", - ] - - spider = FilteredSitemapSpider("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/deutsch/article_1/" - ] - - def test_sitemapindex_filter(self): - sitemap = b""" - - - http://www.example.com/sitemap1.xml - 2004-01-01T20:00:00+00:00 - - - http://www.example.com/sitemap2.xml - 2005-01-01 - - """ - - class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, entries): - for entry in entries: - date_time = datetime.strptime( - entry["lastmod"].split("T")[0], "%Y-%m-%d" - ) - if date_time.year > 2004: - yield entry - - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/sitemap1.xml", - "http://www.example.com/sitemap2.xml", - ] - - spider = FilteredSitemapSpider("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/sitemap2.xml" - ] - - def test_compression_bomb_setting(self): - settings = {"DOWNLOAD_MAXSIZE": 10_000_000} - crawler = get_crawler(settings_dict=settings) - spider = self.spider_class.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request(url="https://example.com") - response = Response(url="https://example.com", body=body, request=request) - assert spider._get_sitemap_body(response) is None - - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_compression_bomb_spider_attr(self): - class DownloadMaxSizeSpider(self.spider_class): - download_maxsize = 10_000_000 - - crawler = get_crawler() - spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request(url="https://example.com") - response = Response(url="https://example.com", body=body, request=request) - assert spider._get_sitemap_body(response) is None - - def test_compression_bomb_request_meta(self): - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request( - url="https://example.com", meta={"download_maxsize": 10_000_000} - ) - response = Response(url="https://example.com", body=body, request=request) - assert spider._get_sitemap_body(response) is None - - def test_download_warnsize_setting(self): - settings = {"DOWNLOAD_WARNSIZE": 10_000_000} - crawler = get_crawler(settings_dict=settings) - spider = self.spider_class.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request(url="https://example.com") - response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: - spider._get_sitemap_body(response) - log.check( - ( - "scrapy.spiders.sitemap", - "WARNING", - ( - "<200 https://example.com> body size after decompression " - "(11511612 B) is larger than the download warning size " - "(10000000 B)." - ), - ), - ) - - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr(self): - class DownloadWarnSizeSpider(self.spider_class): - download_warnsize = 10_000_000 - - crawler = get_crawler() - spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request( - url="https://example.com", meta={"download_warnsize": 10_000_000} - ) - response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: - spider._get_sitemap_body(response) - log.check( - ( - "scrapy.spiders.sitemap", - "WARNING", - ( - "<200 https://example.com> body size after decompression " - "(11511612 B) is larger than the download warning size " - "(10000000 B)." - ), - ), - ) - - def test_download_warnsize_request_meta(self): - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request( - url="https://example.com", meta={"download_warnsize": 10_000_000} - ) - response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: - spider._get_sitemap_body(response) - log.check( - ( - "scrapy.spiders.sitemap", - "WARNING", - ( - "<200 https://example.com> body size after decompression " - "(11511612 B) is larger than the download warning size " - "(10000000 B)." - ), - ), - ) - - @coroutine_test - async def test_sitemap_urls(self): - class TestSpider(self.spider_class): - name = "test" - sitemap_urls = ["https://toscrape.com/sitemap.xml"] - - crawler = get_crawler(TestSpider) - spider = TestSpider.from_crawler(crawler) - with warnings.catch_warnings(): - warnings.simplefilter("error") - requests = [request async for request in spider.start()] - - assert len(requests) == 1 - request = requests[0] - assert request.url == "https://toscrape.com/sitemap.xml" - assert request.dont_filter is False - assert request.callback == spider._parse_sitemap - - -class TestDeprecation: - def test_crawl_spider(self): - assert issubclass(CrawlSpider, Spider) - assert isinstance(CrawlSpider(name="foo"), Spider) - - class TestNoParseMethodSpider: spider_class = Spider diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py new file mode 100644 index 000000000..05d77912f --- /dev/null +++ b/tests/test_spider_crawl.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import re +import warnings +from logging import ERROR + +from testfixtures import LogCapture +from w3lib.url import safe_url_string + +from scrapy.http import HtmlResponse, Request, TextResponse +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule, Spider +from scrapy.utils.test import get_crawler +from tests.test_spider import TestSpider +from tests.utils.decorators import inline_callbacks_test + + +class TestCrawlSpider(TestSpider): + test_body = b"""Page title + +

Item 12

+
+

About us

+
+
+

This shouldn't be followed

+
+ """ + spider_class = CrawlSpider + + def test_rule_without_link_extractor(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(),) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + def test_process_links(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) + + def dummy_process_links(self, links): + return links + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + def test_process_links_filter(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="filter_process_links"),) + _test_regex = re.compile("nofollow") + + def filter_process_links(self, links): + return [link for link in links if not self._test_regex.search(link.url)] + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 2 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + ] + + def test_process_links_generator(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) + + def dummy_process_links(self, links): + yield from links + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + def test_process_request(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + def process_request_change_domain(request, response): + return request.replace(url=request.url.replace(".org", ".com")) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = ( + Rule(LinkExtractor(), process_request=process_request_change_domain), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.com/somepage/item/12.html", + "http://example.com/about.html", + "http://example.com/nofollow.html", + ] + + def test_process_request_with_response(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + def process_request_meta_response_class(request, response): + request.meta["response_class"] = response.__class__.__name__ + return request + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = ( + Rule( + LinkExtractor(), process_request=process_request_meta_response_class + ), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] + + def test_process_request_instance_method(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_request="process_request_upper"),) + + def process_request_upper(self, request, response): + return request.replace(url=request.url.upper()) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), + safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), + safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), + ] + + def test_process_request_instance_method_with_response(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = ( + Rule( + LinkExtractor(), + process_request="process_request_meta_response_class", + ), + ) + + def process_request_meta_response_class(self, request, response): + request.meta["response_class"] = response.__class__.__name__ + return request + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] + + def test_follow_links_attribute_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + assert hasattr(spider, "_follow_links") + assert spider._follow_links + + settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} + crawler = get_crawler(settings_dict=settings_dict) + spider = self.spider_class.from_crawler(crawler, "example.com") + assert hasattr(spider, "_follow_links") + assert not spider._follow_links + + @inline_callbacks_test + def test_start_url(self): + class TestSpider(self.spider_class): + name = "test" + start_url = "https://www.example.com" + + crawler = get_crawler(TestSpider) + with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: + yield crawler.crawl() + assert "Error while reading start items and requests" in str(log) + assert "did you miss an 's'?" in str(log) + + def test_parse_response_use(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(record=True) as w: + spider = _CrawlSpider() + assert len(w) == 0 + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 1 + + def test_parse_response_override(self): + class _CrawlSpider(CrawlSpider): + def _parse_response(self, response, callback, cb_kwargs, follow=True): + pass + + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(record=True) as w: + assert len(w) == 0 + spider = _CrawlSpider() + assert len(w) == 1 + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 1 + + def test_parse_with_rules(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + + with warnings.catch_warnings(record=True) as w: + spider = _CrawlSpider() + spider.parse_with_rules( + TextResponse(spider.start_urls, body=b""), None, None + ) + assert len(w) == 0 + + +class TestDeprecation: + def test_crawl_spider(self): + assert issubclass(CrawlSpider, Spider) + assert isinstance(CrawlSpider(name="foo"), Spider) diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py new file mode 100644 index 000000000..d2664b19e --- /dev/null +++ b/tests/test_spider_sitemap.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +import gzip +import warnings +from datetime import datetime +from io import BytesIO +from logging import WARNING +from pathlib import Path + +import pytest +from testfixtures import LogCapture + +from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse +from scrapy.spiders import SitemapSpider +from scrapy.utils.test import get_crawler +from tests import tests_datadir +from tests.test_spider import TestSpider +from tests.utils.decorators import coroutine_test + + +class TestSitemapSpider(TestSpider): + spider_class = SitemapSpider + + BODY = b"SITEMAP" + f = BytesIO() + g = gzip.GzipFile(fileobj=f, mode="w+b") + g.write(BODY) + g.close() + GZBODY = f.getvalue() + + def assertSitemapBody(self, response: Response, body: bytes | None) -> None: + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + assert spider._get_sitemap_body(response) == body + + def test_get_sitemap_body(self): + r = XmlResponse(url="http://www.example.com/", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + + r = HtmlResponse(url="http://www.example.com/", body=self.BODY) + self.assertSitemapBody(r, None) + + r = Response(url="http://www.example.com/favicon.ico", body=self.BODY) + self.assertSitemapBody(r, None) + + def test_get_sitemap_body_gzip_headers(self): + r = Response( + url="http://www.example.com/sitemap", + body=self.GZBODY, + headers={"content-type": "application/gzip"}, + request=Request("http://www.example.com/sitemap"), + ) + self.assertSitemapBody(r, self.BODY) + + def test_get_sitemap_body_xml_url(self): + r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + + def test_get_sitemap_body_xml_url_compressed(self): + r = Response( + url="http://www.example.com/sitemap.xml.gz", + body=self.GZBODY, + request=Request("http://www.example.com/sitemap"), + ) + self.assertSitemapBody(r, self.BODY) + + # .xml.gz but body decoded by HttpCompression middleware already + r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + + def test_get_sitemap_urls_from_robotstxt(self): + robots = b"""# Sitemap files +Sitemap: http://example.com/sitemap.xml +Sitemap: http://example.com/sitemap-product-index.xml +Sitemap: HTTP://example.com/sitemap-uppercase.xml +Sitemap: /sitemap-relative-url.xml +""" + + r = TextResponse(url="http://www.example.com/robots.txt", body=robots) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://www.example.com/sitemap-relative-url.xml", + ] + + def test_alternate_url_locs(self): + sitemap = b""" + + + http://www.example.com/english/ + + + + + + """ + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] + + spider.sitemap_alternate_links = True + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/italiano/", + ] + + def test_sitemap_filter(self): + sitemap = b""" + + + http://www.example.com/english/ + 2010-01-01 + + + http://www.example.com/portuguese/ + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") + if date_time.year > 2008: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/portuguese/", + ] + + spider = FilteredSitemapSpider("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] + + def test_sitemap_filter_with_alternate_links(self): + sitemap = b""" + + + http://www.example.com/english/article_1/ + 2010-01-01 + + + + http://www.example.com/english/article_2/ + 2015-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + alternate_links = entry.get("alternate", ()) + for link in alternate_links: + if "/deutsch/" in link: + entry["loc"] = link + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/article_1/", + "http://www.example.com/english/article_2/", + ] + + spider = FilteredSitemapSpider("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/deutsch/article_1/" + ] + + def test_sitemapindex_filter(self): + sitemap = b""" + + + http://www.example.com/sitemap1.xml + 2004-01-01T20:00:00+00:00 + + + http://www.example.com/sitemap2.xml + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + date_time = datetime.strptime( + entry["lastmod"].split("T")[0], "%Y-%m-%d" + ) + if date_time.year > 2004: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap1.xml", + "http://www.example.com/sitemap2.xml", + ] + + spider = FilteredSitemapSpider("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap2.xml" + ] + + def test_compression_bomb_setting(self): + settings = {"DOWNLOAD_MAXSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + assert spider._get_sitemap_body(response) is None + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_compression_bomb_spider_attr(self): + class DownloadMaxSizeSpider(self.spider_class): + download_maxsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + assert spider._get_sitemap_body(response) is None + + def test_compression_bomb_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_maxsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + assert spider._get_sitemap_body(response) is None + + def test_download_warnsize_setting(self): + settings = {"DOWNLOAD_WARNSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_download_warnsize_spider_attr(self): + class DownloadWarnSizeSpider(self.spider_class): + download_warnsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + def test_download_warnsize_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + @coroutine_test + async def test_sitemap_urls(self): + class TestSpider(self.spider_class): + name = "test" + sitemap_urls = ["https://toscrape.com/sitemap.xml"] + + crawler = get_crawler(TestSpider) + spider = TestSpider.from_crawler(crawler) + with warnings.catch_warnings(): + warnings.simplefilter("error") + requests = [request async for request in spider.start()] + + assert len(requests) == 1 + request = requests[0] + assert request.url == "https://toscrape.com/sitemap.xml" + assert request.dont_filter is False + assert request.callback == spider._parse_sitemap