From cec86f216e02072b2e9ed5b032810e061143277c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 27 Jul 2026 15:19:12 +0500 Subject: [PATCH] Remove remaining cross-imports in test modules. (#7782) --- pyproject.toml | 3 + tests/mockserver/http_base.py | 8 +- tests/test_command_check.py | 2 +- tests/test_command_crawl.py | 2 +- tests/test_command_genspider.py | 2 +- tests/test_command_parse.py | 2 +- tests/test_commands.py | 2 +- tests/test_downloader_handler_httpx.py | 2 +- .../test_downloader_handler_twisted_http11.py | 2 +- .../test_downloader_handler_twisted_http2.py | 2 +- tests/test_downloadermiddleware_redirect.py | 4 +- ...test_downloadermiddleware_redirect_base.py | 984 ----------------- ...wnloadermiddleware_redirect_metarefresh.py | 4 +- tests/test_engine.py | 337 +----- tests/test_engine_stop_download_bytes.py | 6 +- tests/test_engine_stop_download_headers.py | 6 +- tests/test_feedexport.py | 141 +-- tests/test_feedexport_batch.py | 2 +- tests/test_feedexport_postprocess.py | 2 +- tests/test_http_request.py | 485 +-------- tests/test_http_request_form.py | 4 +- tests/test_http_request_json.py | 4 +- tests/test_http_response.py | 411 +------- tests/test_http_response_text.py | 4 +- tests/test_pipeline_files.py | 4 +- tests/test_pipeline_media.py | 14 +- tests/test_pqueues.py | 2 +- tests/test_scheduler.py | 30 +- tests/test_spider.py | 123 +-- tests/test_spider_crawl.py | 4 +- tests/test_spider_sitemap.py | 4 +- tests/test_spidermiddleware_process_start.py | 3 +- tests/utils/bases/__init__.py | 0 .../{base_commands.py => bases/commands.py} | 0 .../bases/download_handlers_http.py} | 2 - tests/utils/bases/engine.py | 160 +++ tests/utils/bases/feedexport.py | 151 +++ tests/utils/bases/http_request.py | 490 +++++++++ tests/utils/bases/http_response.py | 417 ++++++++ tests/utils/bases/redirect.py | 988 ++++++++++++++++++ tests/utils/bases/spider.py | 131 +++ tests/utils/downloader.py | 35 + tests/utils/engine.py | 184 ++++ tests/utils/media_pipelines.py | 15 + 44 files changed, 2642 insertions(+), 2536 deletions(-) delete mode 100644 tests/test_downloadermiddleware_redirect_base.py create mode 100644 tests/utils/bases/__init__.py rename tests/utils/{base_commands.py => bases/commands.py} (100%) rename tests/{test_downloader_handlers_http_base.py => utils/bases/download_handlers_http.py} (99%) create mode 100644 tests/utils/bases/engine.py create mode 100644 tests/utils/bases/feedexport.py create mode 100644 tests/utils/bases/http_request.py create mode 100644 tests/utils/bases/http_response.py create mode 100644 tests/utils/bases/redirect.py create mode 100644 tests/utils/bases/spider.py create mode 100644 tests/utils/downloader.py create mode 100644 tests/utils/engine.py create mode 100644 tests/utils/media_pipelines.py diff --git a/pyproject.toml b/pyproject.toml index 5ed07a9ce..ee7f4ecdf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,6 +185,9 @@ module = [ "tests.test_utils_misc.test_return_with_argument_inside_generator", "tests.test_utils_python", "tests.test_utils_request", + "tests.utils.bases.http_request", + "tests.utils.bases.http_response", + "tests.utils.bases.spider", ] check_untyped_defs = false diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py index 7b38409ff..343c79781 100644 --- a/tests/mockserver/http_base.py +++ b/tests/mockserver/http_base.py @@ -20,6 +20,9 @@ if TYPE_CHECKING: from twisted.web import resource + # typing.Self requires Python 3.11 + from typing_extensions import Self + class BaseMockServer(ABC): listen_http: bool = True @@ -39,13 +42,14 @@ class BaseMockServer(ABC): self.http_port: int | None = None self.https_port: int | None = None - def __enter__(self): + def __enter__(self) -> Self: self.proc = Popen( [sys.executable, "-u", "-m", self.module_name, *self.get_additional_args()], stdout=PIPE, env=get_script_run_env(), text=True, ) + assert self.proc.stdout is not None if self.listen_http: http_address = self.proc.stdout.readline().strip() http_parsed = urlparse(http_address) @@ -56,7 +60,7 @@ class BaseMockServer(ABC): self.https_port = https_parsed.port return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_value, traceback) -> None: if self.proc: self.proc.kill() self.proc.communicate() diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 794c6d400..240f44584 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -7,7 +7,7 @@ from unittest import TestCase from unittest.mock import MagicMock, Mock, PropertyMock, call, patch from scrapy.commands.check import Command, TextTestResult -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 6293e973e..70c26e6d0 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index f465d0b30..8bb6a2332 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -6,7 +6,7 @@ from pathlib import Path import pytest -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import call, proc, write_recording_editor diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index a3581d764..9b7131c7a 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -8,7 +8,7 @@ import pytest from scrapy.commands import parse from scrapy.settings import Settings -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import call, proc if TYPE_CHECKING: diff --git a/tests/test_commands.py b/tests/test_commands.py index 9e7d4d5a1..51f98db1b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -15,7 +15,7 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.reactor import _asyncio_reactor_path -from tests.utils.base_commands import TestProjectBase +from tests.utils.bases.commands import TestProjectBase from tests.utils.cmdline import call, proc, write_recording_editor if TYPE_CHECKING: diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index fd50248a7..5ceb93382 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -15,7 +15,7 @@ from scrapy.core.downloader.handlers._httpx import ( HttpxDownloadHandler, ) from scrapy.exceptions import DownloadFailedError -from tests.test_downloader_handlers_http_base import ( +from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, TestHttpsBase, diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 79750a136..32dfd7540 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -11,7 +11,7 @@ from scrapy import Spider from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from tests.test_downloader_handlers_http_base import ( +from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, TestHttpsBase, diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index bea97642e..daa89df58 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -13,7 +13,7 @@ from scrapy import Spider from scrapy.crawler import Crawler from scrapy.exceptions import DownloadFailedError, NotConfigured from scrapy.http import Request -from tests.test_downloader_handlers_http_base import ( +from tests.utils.bases.download_handlers_http import ( TestHttpProxyBase, TestHttpsBase, TestHttpsCustomCiphersBase, diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 28cf8ba34..ef2774a93 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -16,11 +16,11 @@ from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.test_downloadermiddleware_redirect_base import Base +from tests.utils.bases.redirect import TestRedirectBase from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS -class TestRedirectMiddleware(Base.Test): +class TestRedirectMiddleware(TestRedirectBase): mwcls = RedirectMiddleware reason = 302 diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py deleted file mode 100644 index bd8bc796c..000000000 --- a/tests/test_downloadermiddleware_redirect_base.py +++ /dev/null @@ -1,984 +0,0 @@ -from __future__ import annotations - -import pytest - -from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware -from scrapy.exceptions import IgnoreRequest -from scrapy.http import Request, Response -from scrapy.utils.misc import set_environ -from scrapy.utils.test import get_crawler - - -class Base: - class Test: - def test_priority_adjust(self): - req = Request("http://a.example") - rsp = self.get_response(req, "http://a.example/redirected") - req2 = self.mw.process_response(req, rsp) - assert req2.priority > req.priority - - def test_dont_redirect(self): - url = "http://www.example.com/301" - url2 = "http://www.example.com/redirected" - req = Request(url, meta={"dont_redirect": True}) - rsp = self.get_response(req, url2) - - r = self.mw.process_response(req, rsp) - assert isinstance(r, Response) - assert r is rsp - - # Test that it redirects when dont_redirect is False - req = Request(url, meta={"dont_redirect": False}) - rsp = self.get_response(req, url2) - - r = self.mw.process_response(req, rsp) - assert isinstance(r, Request) - - def test_post(self): - url = "http://www.example.com/302" - url2 = "http://www.example.com/redirected2" - req = Request( - url, - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = self.get_response(req, url2) - - req2 = self.mw.process_response(req, rsp) - assert isinstance(req2, Request) - assert req2.url == url2 - assert req2.method == "GET" - assert "Content-Type" not in req2.headers, ( - "Content-Type header must not be present in redirected request" - ) - assert "Content-Length" not in req2.headers, ( - "Content-Length header must not be present in redirected request" - ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://a.example/302") - rsp = self.get_response(req, "/redirected") - - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - assert req.meta["redirect_times"] == 1 - with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://a.example/302", meta={"redirect_ttl": 1}) - rsp = self.get_response(req, "/a") - - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp) - - def test_redirect_urls(self): - req1 = Request("http://a.example/first") - rsp1 = self.get_response(req1, "/redirected") - req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req2, "/redirected2") - req3 = self.mw.process_response(req2, rsp2) - - assert req2.url == "http://a.example/redirected" - assert req2.meta["redirect_urls"] == ["http://a.example/first"] - assert req3.url == "http://a.example/redirected2" - assert req3.meta["redirect_urls"] == [ - "http://a.example/first", - "http://a.example/redirected", - ] - - def test_redirect_reasons(self): - req1 = Request("http://a.example/first") - rsp1 = self.get_response(req1, "/redirected1") - req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req2, "/redirected2") - req3 = self.mw.process_response(req2, rsp2) - assert req2.meta["redirect_reasons"] == [self.reason] - assert req3.meta["redirect_reasons"] == [self.reason, self.reason] - - def test_cross_origin_header_dropping(self): - safe_headers = {"A": "B"} - cookie_header = {"Cookie": "a=b"} - authorization_header = {"Authorization": "Bearer 123456"} - - original_request = Request( - "https://example.com", - headers={**safe_headers, **cookie_header, **authorization_header}, - ) - - # Redirects to the same origin (same scheme, same domain, same port) - # keep all headers. - internal_response = self.get_response( - original_request, "https://example.com/a" - ) - internal_redirect_request = self.mw.process_response( - original_request, internal_response - ) - assert isinstance(internal_redirect_request, Request) - assert original_request.headers == internal_redirect_request.headers - - # Redirects to the same origin (same scheme, same domain, same port) - # keep all headers also when the scheme is http. - http_request = Request( - "http://example.com", - headers={**safe_headers, **cookie_header, **authorization_header}, - ) - http_response = self.get_response(http_request, "http://example.com/a") - http_redirect_request = self.mw.process_response( - http_request, http_response - ) - assert isinstance(http_redirect_request, Request) - assert http_request.headers == http_redirect_request.headers - - # For default ports, whether the port is explicit or implicit does not - # affect the outcome, it is still the same origin. - to_explicit_port_response = self.get_response( - original_request, "https://example.com:443/a" - ) - to_explicit_port_redirect_request = self.mw.process_response( - original_request, to_explicit_port_response - ) - assert isinstance(to_explicit_port_redirect_request, Request) - assert original_request.headers == to_explicit_port_redirect_request.headers - - # For default ports, whether the port is explicit or implicit does not - # affect the outcome, it is still the same origin. - to_implicit_port_response = self.get_response( - original_request, "https://example.com/a" - ) - to_implicit_port_redirect_request = self.mw.process_response( - original_request, to_implicit_port_response - ) - assert isinstance(to_implicit_port_redirect_request, Request) - assert original_request.headers == to_implicit_port_redirect_request.headers - - # A port change drops the Authorization header because the origin - # changes, but keeps the Cookie header because the domain remains the - # same. - different_port_response = self.get_response( - original_request, "https://example.com:8080/a" - ) - different_port_redirect_request = self.mw.process_response( - original_request, different_port_response - ) - assert isinstance(different_port_redirect_request, Request) - assert { - **safe_headers, - **cookie_header, - } == different_port_redirect_request.headers.to_unicode_dict() - - # A domain change drops both the Authorization and the Cookie header. - external_response = self.get_response( - original_request, "https://example.org/a" - ) - external_redirect_request = self.mw.process_response( - original_request, external_response - ) - assert isinstance(external_redirect_request, Request) - assert safe_headers == external_redirect_request.headers.to_unicode_dict() - - # A scheme upgrade (http → https) drops the Authorization header - # because the origin changes, but keeps the Cookie header because the - # domain remains the same. - upgrade_response = self.get_response(http_request, "https://example.com/a") - upgrade_redirect_request = self.mw.process_response( - http_request, upgrade_response - ) - assert isinstance(upgrade_redirect_request, Request) - assert { - **safe_headers, - **cookie_header, - } == upgrade_redirect_request.headers.to_unicode_dict() - - # A scheme downgrade (https → http) drops the Authorization header - # because the origin changes, and the Cookie header because its value - # cannot indicate whether the cookies were secure (HTTPS-only) or not. - # - # Note: If the Cookie header is set by the cookie management - # middleware, as recommended in the docs, the dropping of Cookie on - # scheme downgrade is not an issue, because the cookie management - # middleware will add again the Cookie header to the new request if - # appropriate. - downgrade_response = self.get_response( - original_request, "http://example.com/a" - ) - downgrade_redirect_request = self.mw.process_response( - original_request, downgrade_response - ) - assert isinstance(downgrade_redirect_request, Request) - assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() - - def test_meta_proxy_http_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_http_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_http_to_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_to_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_http_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_http_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_https_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_https_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_proxied_http_to_proxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request2.meta["_auth_proxy"] == "https://b.example" - assert request2.meta["proxy"] == "https://b.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_proxied_http_to_unproxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_unproxied_http_to_proxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request2.meta["_auth_proxy"] == "https://b.example" - assert request2.meta["proxy"] == "https://b.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_unproxied_http_to_unproxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_proxied_https_to_proxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request1.meta["_auth_proxy"] == "https://b.example" - assert request1.meta["proxy"] == "https://b.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request3.meta["_auth_proxy"] == "https://b.example" - assert request3.meta["proxy"] == "https://b.example" - - def test_system_proxy_proxied_https_to_unproxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request1.meta["_auth_proxy"] == "https://b.example" - assert request1.meta["proxy"] == "https://b.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request3.meta["_auth_proxy"] == "https://b.example" - assert request3.meta["proxy"] == "https://b.example" - - def test_system_proxy_unproxied_https_to_proxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_unproxied_https_to_unproxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index b5d39080c..aeae759a0 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -12,7 +12,7 @@ from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler -from tests.test_downloadermiddleware_redirect_base import Base +from tests.utils.bases.redirect import TestRedirectBase from tests.utils.redirect import ( HTTP_SCHEMES, NON_HTTP_SCHEMES, @@ -26,7 +26,7 @@ def meta_refresh_body(url, interval=5): return html.encode("utf-8") -class TestMetaRefreshMiddleware(Base.Test): +class TestMetaRefreshMiddleware(TestRedirectBase): mwcls = MetaRefreshMiddleware reason = "meta refresh" diff --git a/tests/test_engine.py b/tests/test_engine.py index 3ab8f6d1b..87f0265a0 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,115 +1,45 @@ from __future__ import annotations import asyncio -import re import subprocess import sys -from collections import defaultdict -from dataclasses import dataclass from typing import TYPE_CHECKING, Any from unittest.mock import Mock -from urllib.parse import urlparse -import attr import pytest -from itemadapter import ItemAdapter -from pydispatch import dispatcher from testfixtures import LogCapture -from twisted.internet import defer from scrapy import signals from scrapy.core.engine import ExecutionEngine, _Slot from scrapy.core.scheduler import BaseScheduler from scrapy.exceptions import CloseSpider, IgnoreRequest -from scrapy.http import Headers, Request, Response -from scrapy.item import Field, Item -from scrapy.linkextractors import LinkExtractor +from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.defer import ( - _schedule_coro, - deferred_from_coro, - maybe_deferred_to_future, -) -from scrapy.utils.signal import disconnect_all +from scrapy.utils.defer import _schedule_coro, deferred_from_coro from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests import get_testdata +from tests.utils.bases.engine import TestEngineBase from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.engine import ( + AttrsItemsSpider, + CrawlerRun, + DataClassItemsSpider, + DictItemsSpider, + MySpider, +) if TYPE_CHECKING: from collections.abc import AsyncIterator - from twisted.python.failure import Failure - from tests.mockserver.http import MockServer -class MyItem(Item): - name = Field() - url = Field() - price = Field() - - -@attr.s -class AttrsItem: - name = attr.ib(default="") - url = attr.ib(default="") - price = attr.ib(default=0) - - -@dataclass -class DataClassItem: - name: str = "" - url: str = "" - price: int = 0 - - -class MySpider(Spider): - name = "scrapytest.org" - - itemurl_re = re.compile(r"item\d+.html") - name_re = re.compile(r"

(.*?)

", re.MULTILINE) - price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) - - item_cls: type = MyItem - - def parse(self, response): - xlink = LinkExtractor() - itemre = re.compile(self.itemurl_re) - for link in xlink.extract_links(response): - if itemre.search(link.url): - yield Request(url=link.url, callback=self.parse_item) - - def parse_item(self, response): - adapter = ItemAdapter(self.item_cls()) - m = self.name_re.search(response.text) - if m: - adapter["name"] = m.group(1) - adapter["url"] = response.url - m = self.price_re.search(response.text) - if m: - adapter["price"] = m.group(1) - return adapter.item - - class DupeFilterSpider(MySpider): async def start(self): for url in self.start_urls: yield Request(url) # no dont_filter=True -class DictItemsSpider(MySpider): - item_cls = dict - - -class AttrsItemsSpider(MySpider): - item_cls = AttrsItem - - -class DataClassItemsSpider(MySpider): - item_cls = DataClassItem - - class ItemZeroDivisionErrorSpider(MySpider): custom_settings = { "ITEM_PIPELINES": { @@ -130,253 +60,6 @@ class ChangeCloseReasonSpider(MySpider): raise CloseSpider(reason="custom_reason") -class CrawlerRun: - """A class to run the crawler and keep track of events occurred""" - - def __init__(self, spider_class: type[Spider]): - self.respplug: list[tuple[Response, Spider]] = [] - self.reqplug: list[tuple[Request, Spider]] = [] - self.reqdropped: list[tuple[Request, Spider]] = [] - self.reqreached: list[tuple[Request, Spider]] = [] - self.itemerror: list[tuple[Any, Response, Spider, Failure]] = [] - self.itemresp: list[tuple[Any, Response]] = [] - self.headers: dict[Request, Headers] = {} - self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list) - self.signals_caught: dict[Any, dict[str, Any]] = {} - self.spider_class = spider_class - - async def run(self, mockserver: MockServer) -> None: - self.mockserver = mockserver - - start_urls = [ - self.geturl("/static/"), - self.geturl("/redirect"), - self.geturl("/redirect"), # duplicate - self.geturl("/numbers"), - ] - - for name, signal in vars(signals).items(): - if not name.startswith("_"): - dispatcher.connect(self.record_signal, signal) - - self.crawler = get_crawler(self.spider_class) - self.crawler.signals.connect(self.item_scraped, signals.item_scraped) - self.crawler.signals.connect(self.item_error, signals.item_error) - self.crawler.signals.connect(self.headers_received, signals.headers_received) - self.crawler.signals.connect(self.bytes_received, signals.bytes_received) - self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) - self.crawler.signals.connect(self.request_dropped, signals.request_dropped) - self.crawler.signals.connect( - self.request_reached, signals.request_reached_downloader - ) - self.crawler.signals.connect( - self.response_downloaded, signals.response_downloaded - ) - self.crawler.crawl(start_urls=start_urls) - - self.deferred: defer.Deferred[None] = defer.Deferred() - dispatcher.connect(self.stop, signals.engine_stopped) - await maybe_deferred_to_future(self.deferred) - - async def stop(self): - for name, signal in vars(signals).items(): - if not name.startswith("_"): - disconnect_all(signal) - self.deferred.callback(None) - await self.crawler.stop_async() - - def geturl(self, path: str) -> str: - return self.mockserver.url(path) - - def getpath(self, url: str) -> str: - u = urlparse(url) - return u.path - - def item_error( - self, item: Any, response: Response, spider: Spider, failure: Failure - ) -> None: - self.itemerror.append((item, response, spider, failure)) - - def item_scraped(self, item: Any, spider: Spider, response: Response) -> None: - self.itemresp.append((item, response)) - - def headers_received( - self, headers: Headers, body_length: int, request: Request, spider: Spider - ) -> None: - self.headers[request] = headers - - def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: - self.bytes[request].append(data) - - def request_scheduled(self, request: Request, spider: Spider) -> None: - self.reqplug.append((request, spider)) - - def request_reached(self, request: Request, spider: Spider) -> None: - self.reqreached.append((request, spider)) - - def request_dropped(self, request: Request, spider: Spider) -> None: - self.reqdropped.append((request, spider)) - - def response_downloaded(self, response: Response, spider: Spider) -> None: - self.respplug.append((response, spider)) - - def record_signal(self, *args: Any, **kwargs: Any) -> None: - """Record a signal and its parameters""" - signalargs = kwargs.copy() - sig = signalargs.pop("signal") - signalargs.pop("sender", None) - self.signals_caught[sig] = signalargs - - -class TestEngineBase: - @staticmethod - def _assert_visited_urls(run: CrawlerRun) -> None: - must_be_visited = [ - "/static/", - "/redirect", - "/redirected", - "/static/item1.html", - "/static/item2.html", - "/static/item999.html", - ] - urls_visited = {rp[0].url for rp in run.respplug} - urls_expected = {run.geturl(p) for p in must_be_visited} - assert urls_expected <= urls_visited, ( - f"URLs not visited: {list(urls_expected - urls_visited)}" - ) - - @staticmethod - def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None: - assert len(run.reqplug) == count - - paths_expected = [ - "/static/item999.html", - "/static/item2.html", - "/static/item1.html", - ] - - urls_requested = {rq[0].url for rq in run.reqplug} - urls_expected = {run.geturl(p) for p in paths_expected} - assert urls_expected <= urls_requested - scheduled_requests_count = len(run.reqplug) - dropped_requests_count = len(run.reqdropped) - responses_count = len(run.respplug) - assert scheduled_requests_count == dropped_requests_count + responses_count - assert len(run.reqreached) == responses_count - - @staticmethod - def _assert_dropped_requests(run: CrawlerRun) -> None: - assert len(run.reqdropped) == 1 - - @staticmethod - def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None: - # response tests - assert len(run.respplug) == count - assert len(run.reqreached) == count - - for response, _ in run.respplug: - if run.getpath(response.url) == "/static/item999.html": - assert response.status == 404 - if run.getpath(response.url) == "/redirect": - assert response.status == 302 - - @staticmethod - def _assert_items_error(run: CrawlerRun) -> None: - assert len(run.itemerror) == 2 - for item, response, spider, failure in run.itemerror: - assert failure.value.__class__ is ZeroDivisionError - assert spider == run.crawler.spider - - assert item["url"] == response.url - if "item1.html" in item["url"]: - assert item["name"] == "Item 1 name" - assert item["price"] == "100" - if "item2.html" in item["url"]: - assert item["name"] == "Item 2 name" - assert item["price"] == "200" - - @staticmethod - def _assert_scraped_items(run: CrawlerRun) -> None: - assert len(run.itemresp) == 2 - for item_, response in run.itemresp: - item = ItemAdapter(item_) - assert item["url"] == response.url - if "item1.html" in item["url"]: - assert item["name"] == "Item 1 name" - assert item["price"] == "100" - if "item2.html" in item["url"]: - assert item["name"] == "Item 2 name" - assert item["price"] == "200" - - @staticmethod - def _assert_headers_received(run: CrawlerRun) -> None: - for headers in run.headers.values(): - assert b"Server" in headers - assert headers[b"Server"] - assert b"TwistedWeb" in headers[b"Server"] - assert b"Date" in headers - assert b"Content-Type" in headers - - @staticmethod - def _assert_bytes_received(run: CrawlerRun) -> None: - assert len(run.bytes) == 9 - for request, data in run.bytes.items(): - joined_data = b"".join(data) - if run.getpath(request.url) == "/static/": - assert joined_data == get_testdata("test_site", "index.html") - elif run.getpath(request.url) == "/static/item1.html": - assert joined_data == get_testdata("test_site", "item1.html") - elif run.getpath(request.url) == "/static/item2.html": - assert joined_data == get_testdata("test_site", "item2.html") - elif run.getpath(request.url) == "/redirected": - assert joined_data == b"Redirected here" - elif run.getpath(request.url) == "/redirect": - assert ( - joined_data == b"\n\n" - b" \n" - b' \n' - b" \n" - b' \n' - b' click here\n' - b" \n" - b"\n" - ) - elif run.getpath(request.url) == "/static/item999.html": - assert ( - joined_data == b"\n\n" - b" 404 - No Such Resource\n" - b" \n" - b"

No Such Resource

\n" - b"

File not found.

\n" - b" \n" - b"\n" - ) - elif run.getpath(request.url) == "/numbers": - # signal was fired multiple times - assert len(data) > 1 - # bytes were received in order - numbers = [str(x).encode("utf8") for x in range(2**18)] - assert joined_data == b"".join(numbers) - - @staticmethod - def _assert_signals_caught(run: CrawlerRun) -> None: - assert signals.engine_started in run.signals_caught - assert signals.engine_stopped in run.signals_caught - assert signals.spider_opened in run.signals_caught - assert signals.spider_idle in run.signals_caught - assert signals.spider_closed in run.signals_caught - assert signals.headers_received in run.signals_caught - - assert {"spider": run.crawler.spider} == run.signals_caught[ - signals.spider_opened - ] - assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] - assert { - "spider": run.crawler.spider, - "reason": "finished", - } == run.signals_caught[signals.spider_closed] - - class TestEngine(TestEngineBase): @coroutine_test async def test_crawler(self, mockserver: MockServer) -> None: diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 091485781..27f282af4 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -3,15 +3,15 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.exceptions import StopDownload -from tests.test_engine import ( +from tests.utils.bases.engine import TestEngineBase +from tests.utils.decorators import coroutine_test +from tests.utils.engine import ( AttrsItemsSpider, CrawlerRun, DataClassItemsSpider, DictItemsSpider, MySpider, - TestEngineBase, ) -from tests.utils.decorators import coroutine_test if TYPE_CHECKING: import pytest diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 8ebc948e2..09fa14326 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -3,15 +3,15 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.exceptions import StopDownload -from tests.test_engine import ( +from tests.utils.bases.engine import TestEngineBase +from tests.utils.decorators import coroutine_test +from tests.utils.engine import ( AttrsItemsSpider, CrawlerRun, DataClassItemsSpider, DictItemsSpider, MySpider, - TestEngineBase, ) -from tests.utils.decorators import coroutine_test if TYPE_CHECKING: import pytest diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3f659ee0b..f1c416fbd 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -4,13 +4,9 @@ import csv import json import marshal import pickle -import random -import shutil import tempfile -from abc import ABC, abstractmethod from logging import getLogger from pathlib import Path -from string import ascii_letters, digits from typing import IO, TYPE_CHECKING, Any from unittest import mock @@ -32,8 +28,8 @@ from scrapy.extensions.feedexport import ( ) from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.spiders import ItemSpider +from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test, inline_callbacks_test from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape @@ -99,141 +95,6 @@ class LogOnStoreFileStorage: file.close() -class TestFeedExportBase(ABC): - mockserver: MockServer - - def _random_temp_filename(self, inter_dir="") -> Path: - chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = "".join(chars) - return Path(self.temp_dir, inter_dir, filename) - - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - def setup_method(self): - self.temp_dir = tempfile.mkdtemp() - - def teardown_method(self): - shutil.rmtree(self.temp_dir, ignore_errors=True) - - async def exported_data( - self, items: Iterable[Any], settings: dict[str, Any] - ) -> dict[str, Any]: - """ - Return exported data which a spider yielding ``items`` would return. - """ - - class TestSpider(scrapy.Spider): - name = "testspider" - - def parse(self, response): - yield from items - - return await self.run_and_export(TestSpider, settings) - - async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]: - """ - Return exported data which a spider yielding no ``items`` would return. - """ - - class TestSpider(scrapy.Spider): - name = "testspider" - - def parse(self, response): - pass - - return await self.run_and_export(TestSpider, settings) - - async def assertExported( - self, - items: Iterable[Any], - header: Iterable[str], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - await self.assertExportedCsv(items, header, rows, settings) - await self.assertExportedJsonLines(items, rows, settings) - await self.assertExportedXml(items, rows, settings) - await self.assertExportedPickle(items, rows, settings) - await self.assertExportedMarshal(items, rows, settings) - await self.assertExportedMultiple(items, rows, settings) - - async def assertExportedCsv( # noqa: B027 - self, - items: Iterable[Any], - header: Iterable[str], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedJsonLines( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedXml( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedMultiple( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedPickle( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - async def assertExportedMarshal( # noqa: B027 - self, - items: Iterable[Any], - rows: Iterable[dict[str, Any]], - settings: dict[str, Any] | None = None, - ) -> None: - pass - - @abstractmethod - async def run_and_export( - self, spider_cls: type[Spider], settings: dict[str, Any] - ) -> dict[str, Any]: - pass - - def _load_until_eof( - self, data: bytes, load_func: Callable[[IO[bytes]], Any] - ) -> list[Any]: - result: list[Any] = [] - with tempfile.TemporaryFile() as temp: - temp.write(data) - temp.seek(0) - while True: - try: - result.append(load_func(temp)) - except EOFError: - break - return result - - class InstrumentedFeedSlot(FeedSlot): """Instrumented FeedSlot subclass for keeping track of calls to start_exporting and finish_exporting.""" diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 3b50cd492..80ff6229b 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -21,7 +21,7 @@ from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler from tests.spiders import ItemSpider -from tests.test_feedexport import TestFeedExportBase +from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test, inline_callbacks_test from tests.utils.feedexport import MyItem diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py index 83d976e94..f120ce36f 100644 --- a/tests/test_feedexport_postprocess.py +++ b/tests/test_feedexport_postprocess.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any import pytest from scrapy.utils.test import get_crawler -from tests.test_feedexport import TestFeedExportBase +from tests.utils.bases.feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test from tests.utils.feedexport import path_to_url, printf_escape diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 1941b826f..e58ae8f39 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,493 +1,18 @@ -import warnings import xmlrpc.client -from typing import Any import pytest -from scrapy.http import Headers, Request, XmlRpcRequest -from scrapy.http.request import NO_CALLBACK +from scrapy import Request +from scrapy.http import XmlRpcRequest from scrapy.utils.python import to_bytes +from tests.utils.bases.http_request import TestRequestBase -class TestRequest: +class TestRequest(TestRequestBase): request_class = Request - default_method = "GET" - default_headers: dict[bytes, list[bytes]] = {} - default_meta: dict[str, Any] = {} - def test_init(self): - # Request requires url in the __init__ method - with pytest.raises(TypeError): - self.request_class() - # url argument must be basestring - with pytest.raises(TypeError): - self.request_class(123) - - # priority argument must be an integer - with pytest.raises(TypeError, match="Request priority not an integer"): - self.request_class("http://www.example.com", priority="1") - - r = self.request_class("http://www.example.com") - assert isinstance(r.url, str) - assert r.url == "http://www.example.com" - assert r.method == self.default_method - - assert isinstance(r.headers, Headers) - assert r.headers == self.default_headers - assert r.meta == self.default_meta - - meta = {"lala": "lolo"} - headers = {b"caca": b"coco"} - r = self.request_class( - "http://www.example.com", meta=meta, headers=headers, body="a body" - ) - - assert r.meta is not meta - assert r.meta == meta - assert r.headers is not headers - assert r.headers[b"caca"] == b"coco" - - def test_url_scheme(self): - # This test passes by not raising any (ValueError) exception - self.request_class("http://example.org") - self.request_class("https://example.org") - self.request_class("s3://example.org") - self.request_class("ftp://example.org") - self.request_class("about:config") - self.request_class("data:,Hello%2C%20World!") - - def test_url_no_scheme(self): - msg = "Missing scheme in request url:" - with pytest.raises(ValueError, match=msg): - self.request_class("foo") - with pytest.raises(ValueError, match=msg): - self.request_class("/foo/") - with pytest.raises(ValueError, match=msg): - self.request_class("/foo:bar") - - def test_headers(self): - # Different ways of setting headers attribute - url = "http://www.scrapy.org" - headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"} - r = self.request_class(url=url, headers=headers) - p = self.request_class(url=url, headers=r.headers) - - assert r.headers == p.headers - assert r.headers is not headers - assert p.headers is not r.headers - - # headers must not be unicode - h = Headers({"key1": "val1", "key2": "val2"}) - h["newkey"] = "newval" - for k, v in h.items(): - assert isinstance(k, bytes) - for s in v: - assert isinstance(s, bytes) - - def test_eq(self): - url = "http://www.scrapy.org" - r1 = self.request_class(url=url) - r2 = self.request_class(url=url) - assert r1 != r2 - - set_ = set() - set_.add(r1) - set_.add(r2) - assert len(set_) == 2 - - def test_url(self): - r = self.request_class(url="http://www.scrapy.org/path") - assert r.url == "http://www.scrapy.org/path" - - def test_url_quoting(self): - r = self.request_class(url="http://www.scrapy.org/blank%20space") - assert r.url == "http://www.scrapy.org/blank%20space" - r = self.request_class(url="http://www.scrapy.org/blank space") - assert r.url == "http://www.scrapy.org/blank%20space" - - def test_url_encoding(self): - r = self.request_class(url="http://www.scrapy.org/price/£") - assert r.url == "http://www.scrapy.org/price/%C2%A3" - - def test_url_encoding_other(self): - # encoding affects only query part of URI, not path - # path part should always be UTF-8 encoded before percent-escaping - r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") - assert r.url == "http://www.scrapy.org/price/%C2%A3" - - r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") - assert r.url == "http://www.scrapy.org/price/%C2%A3" - - def test_url_encoding_query(self): - r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") - assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" - - # should be same as above - r2 = self.request_class( - url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" - ) - assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" - - def test_url_encoding_query_latin1(self): - # encoding is used for encoding query-string before percent-escaping; - # path is still UTF-8 encoded before percent-escaping - r3 = self.request_class( - url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" - ) - assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3" - - def test_url_encoding_nonutf8_untouched(self): - # percent-escaping sequences that do not match valid UTF-8 sequences - # should be kept untouched (just upper-cased perhaps) - # - # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2 - # - # "Conversions from URIs to IRIs MUST NOT use any character encoding - # other than UTF-8 in steps 3 and 4, even if it might be possible to - # guess from the context that another character encoding than UTF-8 was - # used in the URI. For example, the URI - # "http://www.example.org/r%E9sum%E9.html" might with some guessing be - # interpreted to contain two e-acute characters encoded as iso-8859-1. - # It must not be converted to an IRI containing these e-acute - # characters. Otherwise, in the future the IRI will be mapped to - # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different - # URI from "http://www.example.org/r%E9sum%E9.html". - r1 = self.request_class(url="http://www.scrapy.org/price/%a3") - assert r1.url == "http://www.scrapy.org/price/%a3" - - r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" - - r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") - assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" - - r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") - assert r4.url == "http://www.example.org/r%E9sum%E9.html" - - def test_url_verbatim(self): - r = self.request_class( - url="http://www.scrapy.org/price/£", - meta={"verbatim_url": True}, - ) - assert r.url == "http://www.scrapy.org/price/£" - - r = self.request_class( - url="http://www.scrapy.org/blank space", - meta={"verbatim_url": True}, - ) - assert r.url == "http://www.scrapy.org/blank space" - - def test_body(self): - r1 = self.request_class(url="http://www.example.com/") - assert r1.body == b"" - - r2 = self.request_class(url="http://www.example.com/", body=b"") - assert isinstance(r2.body, bytes) - assert r2.encoding == "utf-8" # default encoding - - r3 = self.request_class( - url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" - ) - assert isinstance(r3.body, bytes) - assert r3.body == b"Price: \xc2\xa3100" - - r4 = self.request_class( - url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" - ) - assert isinstance(r4.body, bytes) - assert r4.body == b"Price: \xa3100" - - def test_copy(self): - """Test Request copy""" - - def somecallback(): - pass - - r1 = self.request_class( - "http://www.example.com", - flags=["f1", "f2"], - callback=somecallback, - errback=somecallback, - ) - r1.meta["foo"] = "bar" - r1.cb_kwargs["key"] = "value" - r2 = r1.copy() - - # make sure callbaclks are copied - assert r1.callback is somecallback - assert r1.errback is somecallback - assert r2.callback is r1.callback - assert r2.errback is r1.errback - - # make sure flags list is shallow copied - assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - assert r1.flags == r2.flags - - # make sure cb_kwargs dict is shallow copied - assert r1.cb_kwargs is not r2.cb_kwargs, ( - "cb_kwargs must be a shallow copy, not identical" - ) - assert r1.cb_kwargs == r2.cb_kwargs - - # make sure meta dict is shallow copied - assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" - assert r1.meta == r2.meta - - # make sure headers attribute is shallow copied - assert r1.headers is not r2.headers, ( - "headers must be a shallow copy, not identical" - ) - assert r1.headers == r2.headers - assert r1.encoding == r2.encoding - assert r1.dont_filter == r2.dont_filter - - # Request.body can be identical since it's an immutable object (str) - - def test_copy_inherited_classes(self): - """Test Request children copies preserve their class""" - - class CustomRequest(self.request_class): - pass - - r1 = CustomRequest("http://www.example.com") - r2 = r1.copy() - - assert isinstance(r2, CustomRequest) - - def test_replace(self): - """Test Request.replace() method""" - r1 = self.request_class("http://www.example.com", method="GET") - hdrs = Headers(r1.headers) - hdrs[b"key"] = b"value" - r2 = r1.replace(method="POST", body="New body", headers=hdrs) - assert r1.url == r2.url - assert (r1.method, r2.method) == ("GET", "POST") - assert (r1.body, r2.body) == (b"", b"New body") - assert (r1.headers, r2.headers) == (self.default_headers, hdrs) - - # Empty attributes (which may fail if not compared properly) - r3 = self.request_class( - "http://www.example.com", meta={"a": 1}, dont_filter=True - ) - r4 = r3.replace( - url="http://www.example.com/2", body=b"", meta={}, dont_filter=False - ) - assert r4.url == "http://www.example.com/2" - assert r4.body == b"" - assert r4.meta == {} - assert r4.dont_filter is False - - # the cls argument allows changing the resulting class - custom_request_cls = type("CustomRequest", (self.request_class,), {}) - r5 = r1.replace(cls=custom_request_cls) - assert isinstance(r5, custom_request_cls) - assert r5.url == r1.url - - def test_method_always_str(self): - r = self.request_class("http://www.example.com", method="POST") - assert isinstance(r.method, str) - - def test_immutable_attributes(self): - r = self.request_class("http://example.com") - with pytest.raises(AttributeError): - r.url = "http://example2.com" - with pytest.raises(AttributeError): - r.body = "xxx" - - def test_callback_and_errback(self): - def a_function(): - pass - - r1 = self.request_class("http://example.com") - assert r1.callback is None - assert r1.errback is None - - r2 = self.request_class("http://example.com", callback=a_function) - assert r2.callback is a_function - assert r2.errback is None - - r3 = self.request_class("http://example.com", errback=a_function) - assert r3.callback is None - assert r3.errback is a_function - - r4 = self.request_class( - url="http://example.com", - callback=a_function, - errback=a_function, - ) - assert r4.callback is a_function - assert r4.errback is a_function - - r5 = self.request_class( - url="http://example.com", - callback=NO_CALLBACK, - errback=NO_CALLBACK, - ) - assert r5.callback is NO_CALLBACK - assert r5.errback is NO_CALLBACK - - def test_callback_and_errback_type(self): - with pytest.raises(TypeError): - self.request_class("http://example.com", callback="a_function") - with pytest.raises(TypeError): - self.request_class("http://example.com", errback="a_function") - with pytest.raises(TypeError): - self.request_class( - url="http://example.com", - callback="a_function", - errback="a_function", - ) - - def test_setters(self): - request = self.request_class("http://example.com") - - request.flags = ["f1"] - assert request.flags == ["f1"] - - request.cookies = {"sid": "1"} - assert request.cookies == {"sid": "1"} - - headers = Headers({b"X-Test": b"1"}) - request.headers = headers - assert request._headers is headers - request.headers = {b"A": b"b"} - assert isinstance(request.headers, Headers) - assert request._headers[b"A"] == b"b" - - def test_setter_mutable_lazy_loading(self): - """Mutable attributes are set internally to None only until they are - read, then they always return the same falsy instance of the - corresponding mutable structure. - - Setting them to None causes the next read to return a different object. - """ - - request = self.request_class("http://example.com") - - assert request._flags is None - assert request.flags == [] - assert request.flags is request.flags - assert request._flags == [] - original_flags = request.flags - request.flags = None - assert request._flags is None - assert request.flags == [] - assert request.flags is not original_flags - - assert request._cookies is None - assert request.cookies == {} - assert request.cookies is request.cookies - assert request._cookies == {} - original_cookies = request.cookies - request.cookies = None - assert request._cookies is None - assert request.cookies == {} - assert request.cookies is not original_cookies - - if self.default_headers: - assert request._headers == self.default_headers - assert request._headers is not self.default_headers - assert request.headers == self.default_headers - else: - assert request._headers is None - assert request.headers == {} - assert request.headers is request.headers - assert isinstance(request.headers, Headers) - assert isinstance(request._headers, Headers) - original_headers = request.headers - request.headers = None - assert request._headers is None - assert request.headers == {} - assert request._headers == {} - assert request.headers is not original_headers - - def test_no_callback(self): - with pytest.raises(RuntimeError): - NO_CALLBACK() - - def test_from_curl(self): - # Note: more curated tests regarding curl conversion are in - # `test_utils_curl.py` - curl_command = ( - "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique" - "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique" - "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H" - " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q" - "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use" - "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM" - "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S" - "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode" - "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=" - "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0" - "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee" - "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith" - "%40example.org&size=small&topping=cheese&topping=onion&delivery=1" - "2%3A15&comments=' --compressed" - ) - r = self.request_class.from_curl(curl_command) - assert r.method == "POST" - assert r.url == "http://httpbin.org/post" - assert ( - r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40" - b"example.org&size=small&topping=cheese&topping=onion" - b"&delivery=12%3A15&comments=" - ) - assert r.cookies == { - "_gauges_unique_year": "1", - "_gauges_unique": "1", - "_gauges_unique_month": "1", - "_gauges_unique_hour": "1", - "_gauges_unique_day": "1", - } - assert r.headers == { - b"Origin": [b"http://httpbin.org"], - b"Accept-Encoding": [b"gzip, deflate"], - b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], - b"Upgrade-Insecure-Requests": [b"1"], - b"User-Agent": [ - b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." - b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" - b".75 Chrome/62.0.3202.75 Safari/537.36" - ], - b"Content-Type": [b"application /x-www-form-urlencoded"], - b"Accept": [ - b"text/html,application/xhtml+xml,application/xml;q=0." - b"9,image/webp,image/apng,*/*;q=0.8" - ], - b"Cache-Control": [b"max-age=0"], - b"Referer": [b"http://httpbin.org/forms/post"], - b"Connection": [b"keep-alive"], - } - - def test_from_curl_with_kwargs(self): - r = self.request_class.from_curl( - 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} - ) - assert r.method == "POST" - assert r.meta == {"key": "value"} - - def test_from_curl_ignore_unknown_options(self): - # By default: it works and ignores the unknown options: --foo and -z - with warnings.catch_warnings(): # avoid warning when executing tests - warnings.filterwarnings( - "ignore", category=UserWarning, message="Unrecognized options:" - ) - r = self.request_class.from_curl( - 'curl -X DELETE "http://example.org" --foo -z', - ) - assert r.method == "DELETE" - - # If `ignore_unknown_options` is set to `False` it raises an error with - # the unknown options: --foo and -z - with pytest.raises(ValueError, match="Unrecognized options:"): - self.request_class.from_curl( - 'curl -X PATCH "http://example.org" --foo -z', - ignore_unknown_options=False, - ) - - -class TestXmlRpcRequest(TestRequest): +class TestXmlRpcRequest(TestRequestBase): request_class = XmlRpcRequest default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index af86b35c0..5e965e8dc 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -10,7 +10,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, HtmlResponse from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode -from tests.test_http_request import TestRequest +from tests.utils.bases.http_request import TestRequestBase def _buildresponse(body, **kwargs): @@ -31,7 +31,7 @@ def _qs(req, encoding="utf-8", to_unicode=False): # FormRequest.from_response() is deprecated in favor of form2request, so the # many tests below that exercise it ignore the resulting deprecation warning. @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestFormRequest(TestRequest): +class TestFormRequest(TestRequestBase): request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): diff --git a/tests/test_http_request_json.py b/tests/test_http_request_json.py index 65022afe1..1bfbbbdf5 100644 --- a/tests/test_http_request_json.py +++ b/tests/test_http_request_json.py @@ -8,10 +8,10 @@ import pytest from scrapy.http import JsonRequest from scrapy.utils.python import to_bytes -from tests.test_http_request import TestRequest +from tests.utils.bases.http_request import TestRequestBase -class TestJsonRequest(TestRequest): +class TestJsonRequest(TestRequestBase): request_class = JsonRequest default_method = "GET" default_headers = { diff --git a/tests/test_http_response.py b/tests/test_http_response.py index a8ea4920e..369c2495f 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,413 +1,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING - -import pytest -from w3lib.encoding import resolve_encoding - -from scrapy.exceptions import NotSupported -from scrapy.http import Headers, Request, Response -from scrapy.link import Link -from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS -from tests import get_testdata - -if TYPE_CHECKING: - from collections.abc import Iterable +from scrapy.http import Response +from tests.utils.bases.http_response import TestResponseBase -class TestResponse: +class TestResponse(TestResponseBase): response_class = Response - - def test_init(self): - # Response requires url in the constructor - with pytest.raises(TypeError): - self.response_class() - assert isinstance( - self.response_class("http://example.com/"), self.response_class - ) - with pytest.raises(TypeError): - self.response_class(b"http://example.com") - with pytest.raises(TypeError): - self.response_class(url="http://example.com", body={}) - # body can be str or None - assert isinstance( - self.response_class("http://example.com/", body=b""), - self.response_class, - ) - assert isinstance( - self.response_class("http://example.com/", body=b"body"), - self.response_class, - ) - # test presence of all optional parameters - assert isinstance( - self.response_class( - "http://example.com/", body=b"", headers={}, status=200 - ), - self.response_class, - ) - - r = self.response_class("http://www.example.com") - assert isinstance(r.url, str) - assert r.url == "http://www.example.com" - assert r.status == 200 - - assert isinstance(r.headers, Headers) - assert not r.headers - - headers = {"foo": "bar"} - body = b"a body" - r = self.response_class("http://www.example.com", headers=headers, body=body) - - assert r.headers is not headers - assert r.headers[b"foo"] == b"bar" - - r = self.response_class("http://www.example.com", status=301) - assert r.status == 301 - r = self.response_class("http://www.example.com", status="301") - assert r.status == 301 - with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): - self.response_class("http://example.com", status="lala200") - - def test_copy(self): - """Test Response copy""" - - r1 = self.response_class("http://www.example.com", body=b"Some body") - r1.flags.append("cached") - r2 = r1.copy() - - assert r1.status == r2.status - assert r1.body == r2.body - - # make sure flags list is shallow copied - assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" - assert r1.flags == r2.flags - - # make sure headers attribute is shallow copied - assert r1.headers is not r2.headers, ( - "headers must be a shallow copy, not identical" - ) - assert r1.headers == r2.headers - - def test_copy_meta(self): - req = Request("http://www.example.com") - req.meta["foo"] = "bar" - r1 = self.response_class( - "http://www.example.com", body=b"Some body", request=req - ) - assert r1.meta is req.meta - - def test_copy_cb_kwargs(self): - req = Request("http://www.example.com") - req.cb_kwargs["foo"] = "bar" - r1 = self.response_class( - "http://www.example.com", body=b"Some body", request=req - ) - assert r1.cb_kwargs is req.cb_kwargs - - def test_unavailable_meta(self): - r1 = self.response_class("http://www.example.com", body=b"Some body") - with pytest.raises(AttributeError, match=r"Response\.meta not available"): - r1.meta - - def test_unavailable_cb_kwargs(self): - r1 = self.response_class("http://www.example.com", body=b"Some body") - with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"): - r1.cb_kwargs - - def test_copy_inherited_classes(self): - """Test Response children copies preserve their class""" - - class CustomResponse(self.response_class): - pass - - r1 = CustomResponse("http://www.example.com") - r2 = r1.copy() - - assert isinstance(r2, CustomResponse) - - def test_replace(self): - """Test Response.replace() method""" - hdrs = Headers({"key": "value"}) - r1 = self.response_class("http://www.example.com") - r2 = r1.replace(status=301, body=b"New body", headers=hdrs) - assert r1.body == b"" - assert r1.url == r2.url - assert (r1.status, r2.status) == (200, 301) - assert (r1.body, r2.body) == (b"", b"New body") - assert (r1.headers, r2.headers) == ({}, hdrs) - - # Empty attributes (which may fail if not compared properly) - r3 = self.response_class("http://www.example.com", flags=["cached"]) - r4 = r3.replace(body=b"", flags=[]) - assert r4.body == b"" - assert not r4.flags - - def _assert_response_values(self, response, encoding, body): - if isinstance(body, str): - body_unicode = body - body_bytes = body.encode(encoding) - else: - body_unicode = body.decode(encoding) - body_bytes = body - - assert isinstance(response.body, bytes) - assert isinstance(response.text, str) - self._assert_response_encoding(response, encoding) - assert response.body == body_bytes - assert response.text == body_unicode - - def _assert_response_encoding(self, response, encoding): - assert response.encoding == resolve_encoding(encoding) - - def test_immutable_attributes(self): - r = self.response_class("http://example.com") - with pytest.raises(AttributeError): - r.url = "http://example2.com" - with pytest.raises(AttributeError): - r.body = "xxx" - - def test_setter_mutable_lazy_loading(self): - """Mutable attributes are set internally to None only until they are - read, then they always return the same falsy instance of the - corresponding mutable structure. - - Setting them to None causes the next read to return a different object. - """ - - response = self.response_class("http://example.com") - - response.request = Request("http://example.com") - - assert response._flags is None - assert response.flags == [] - assert response.flags is response.flags - assert response._flags == [] - original_flags = response.flags - response.flags = None - assert response._flags is None - assert response.flags == [] - assert response.flags is not original_flags - - assert response._headers is None - assert response.headers == {} - assert response.headers is response.headers - assert isinstance(response.headers, Headers) - assert isinstance(response._headers, Headers) - original_headers = response.headers - response.headers = None - assert response._headers is None - assert response.headers == {} - assert response._headers == {} - assert response.headers is not original_headers - - def test_setters(self): - response = self.response_class("http://example.com") - - response.flags = ["f1"] - assert response.flags == ["f1"] - - headers = Headers({b"X-Test": b"1"}) - response.headers = headers - assert response._headers is headers - response.headers = {b"A": b"b"} - assert isinstance(response.headers, Headers) - assert response._headers[b"A"] == b"b" - - def test_urljoin(self): - """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" - joined = self.response_class("http://www.example.com").urljoin("/test") - absolute = "http://www.example.com/test" - assert joined == absolute - - def test_shortcut_attributes(self): - r = self.response_class("http://example.com", body=b"hello") - if self.response_class == Response: - msg = "Response content isn't text" - with pytest.raises(AttributeError, match=msg): - r.text - with pytest.raises(NotSupported, match=msg): - r.css("body") - with pytest.raises(NotSupported, match=msg): - r.xpath("//body") - with pytest.raises(NotSupported, match=msg): - r.jmespath("body") - else: - r.text - r.css("body") - r.xpath("//body") - - # Response.follow - - def test_follow_url_absolute(self): - self._assert_followed_url("http://foo.example.com", "http://foo.example.com") - - def test_follow_url_relative(self): - self._assert_followed_url("foo", "http://example.com/foo") - - def test_follow_link(self): - self._assert_followed_url( - Link("http://example.com/foo"), "http://example.com/foo" - ) - - def test_follow_None_url(self): - r = self.response_class("http://example.com") - with pytest.raises(ValueError, match="url can't be None"): - r.follow(None) - - def test_follow_None_encoding(self): - r = self.response_class("http://example.com") - with pytest.raises(ValueError, match="encoding can't be None"): - r.follow("foo", encoding=None) - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_whitespace_url(self): - self._assert_followed_url("foo ", "http://example.com/foo") - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_whitespace_link(self): - self._assert_followed_url( - Link("http://example.com/foo "), "http://example.com/foo" - ) - - def test_follow_flags(self): - res = self.response_class("http://example.com/") - fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - assert fol.flags == ["cached", "allowed"] - - # Response.follow_all - - def test_follow_all_absolute(self): - url_list = [ - "http://example.org", - "http://www.example.org", - "http://example.com", - "http://www.example.com", - ] - self._assert_followed_all_urls(url_list, url_list) - - def test_follow_all_relative(self): - relative = ["foo", "bar", "foo/bar", "bar/foo"] - absolute = [ - "http://example.com/foo", - "http://example.com/bar", - "http://example.com/foo/bar", - "http://example.com/bar/foo", - ] - self._assert_followed_all_urls(relative, absolute) - - def test_follow_all_links(self): - absolute = [ - "http://example.com/foo", - "http://example.com/bar", - "http://example.com/foo/bar", - "http://example.com/bar/foo", - ] - links = map(Link, absolute) - self._assert_followed_all_urls(links, absolute) - - def test_follow_all_empty(self): - r = self.response_class("http://example.com") - assert not list(r.follow_all([])) - - def test_follow_all_invalid(self): - r = self.response_class("http://example.com") - if self.response_class == Response: - with pytest.raises(TypeError): - list(r.follow_all(urls=None)) - with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) - with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) - else: - with pytest.raises( - ValueError, match="Please supply exactly one of the following arguments" - ): - list(r.follow_all(urls=None)) - with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) - with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_all_whitespace(self): - relative = ["foo ", "bar ", "foo/bar ", "bar/foo "] - absolute = [ - "http://example.com/foo", - "http://example.com/bar", - "http://example.com/foo/bar", - "http://example.com/bar/foo", - ] - self._assert_followed_all_urls(relative, absolute) - - @pytest.mark.xfail( - not W3LIB_STRIPS_URLS, - reason="https://github.com/scrapy/w3lib/pull/207", - strict=True, - ) - def test_follow_all_whitespace_links(self): - absolute = [ - "http://example.com/foo ", - "http://example.com/bar ", - "http://example.com/foo/bar ", - "http://example.com/bar/foo ", - ] - links = [Link(u) for u in absolute] - expected = [u.strip() for u in absolute] - self._assert_followed_all_urls(links, expected) - - def test_follow_all_flags(self): - re = self.response_class("http://www.example.com/") - urls = [ - "http://www.example.com/", - "http://www.example.com/2", - "http://www.example.com/foo", - ] - fol = re.follow_all(urls, flags=["cached", "allowed"]) - for req in fol: - assert req.flags == ["cached", "allowed"] - - def _assert_followed_url( - self, - follow_obj: str | Link, - target_url: str, - response: Response | None = None, - encoding: str | None = None, - ) -> None: - if response is None: - response = self._links_response() - req = response.follow(follow_obj) - assert req.url == target_url - if encoding is not None: - assert req.encoding == encoding - - def _assert_followed_all_urls( - self, - follow_obj: Iterable[str | Link], - target_urls: Iterable[str], - response: Response | None = None, - ) -> None: - if response is None: - response = self._links_response() - followed = response.follow_all(follow_obj) - for req, target in zip(followed, target_urls, strict=True): - assert req.url == target - - def _links_response(self) -> Response: - body = get_testdata("link_extractor", "linkextractor.html") - return self.response_class("http://example.com/index", body=body) - - def _links_response_no_href(self) -> Response: - body = get_testdata("link_extractor", "linkextractor_no_href.html") - return self.response_class("http://example.com/index", body=body) diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index daf6e66e5..04315ad89 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -8,10 +8,10 @@ import pytest from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector from scrapy.utils.python import to_unicode -from tests.test_http_response import TestResponse +from tests.utils.bases.http_response import TestResponseBase -class TestTextResponse(TestResponse): +class TestTextResponse(TestResponseBase): response_class = TextResponse def test_follow_None_encoding(self): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 0d9aac830..a0ae3635b 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -39,8 +39,8 @@ from scrapy.utils.test import get_crawler from tests.mockserver.ftp import MockFTPServer from tests.utils.decorators import coroutine_test, inline_callbacks_test -from .test_pipeline_media import _mocked_download_func from .utils.cloud import mock_google_cloud_storage +from .utils.media_pipelines import mocked_download_func # required by persist_file() and stat_file(), but as some stores don't use the argument # we can pass this singleton to keep type hints correct @@ -94,7 +94,7 @@ class TestFilesPipeline: settings_dict = {"FILES_STORE": self.tempdir} crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) crawler.spider = crawler._create_spider() - crawler.engine = MagicMock(download_async=_mocked_download_func) + crawler.engine = MagicMock(download_async=mocked_download_func) self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline.open_spider() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 6862c34eb..ee7a576db 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -9,7 +9,6 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException from scrapy.pipelines.media import MediaPipeline from scrapy.utils.defer import _defer_sleep_async @@ -18,16 +17,7 @@ from scrapy.utils.signal import disconnect_all from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test - - -async def _mocked_download_func(request): - assert request.callback is NO_CALLBACK - response = request.meta.get("response") - if callable(response): - response = await response() - if isinstance(response, Exception): - raise response - return response +from tests.utils.media_pipelines import mocked_download_func class UserDefinedPipeline(MediaPipeline): @@ -54,7 +44,7 @@ class TestBaseMediaPipeline: def setup_method(self): crawler = get_crawler(DefaultSpider, self.settings) crawler.spider = crawler._create_spider() - crawler.engine = MagicMock(download_async=_mocked_download_func) + crawler.engine = MagicMock(download_async=mocked_download_func) self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.open_spider() self.info = self.pipe.spiderinfo diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 6c6a6584a..85fefd172 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -11,7 +11,7 @@ from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler -from tests.test_scheduler import MockDownloader +from tests.utils.downloader import MockDownloader class TestPriorityQueue: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 4fb594444..d7c21bdbd 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -3,7 +3,7 @@ from __future__ import annotations import warnings from abc import ABC, abstractmethod from contextlib import AbstractAsyncContextManager, asynccontextmanager -from typing import TYPE_CHECKING, Any, NamedTuple, cast +from typing import TYPE_CHECKING from unittest.mock import Mock import pytest @@ -15,43 +15,17 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.defer import ensure_awaitable -from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.downloader import MockDownloader if TYPE_CHECKING: from collections.abc import AsyncGenerator from pathlib import Path -class MockSlot(NamedTuple): - active: list[Any] - - -class MockDownloader: - def __init__(self) -> None: - self.slots: dict[str, MockSlot] = {} - - def get_slot_key(self, request: Request) -> str: - if Downloader.DOWNLOAD_SLOT in request.meta: - return cast("str", request.meta[Downloader.DOWNLOAD_SLOT]) - - return urlparse_cached(request).hostname or "" - - def increment(self, slot_key: str) -> None: - slot = self.slots.setdefault(slot_key, MockSlot(active=[])) - slot.active.append(1) - - def decrement(self, slot_key: str) -> None: - slot = self.slots[slot_key] - slot.active.pop() - - def close(self) -> None: - pass - - class MockCrawler(Crawler): def __init__(self, priority_queue_cls: str, jobdir: Path | None): settings = { diff --git a/tests/test_spider.py b/tests/test_spider.py index 7b2727370..03d17199f 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,133 +1,18 @@ from __future__ import annotations -from typing import Any -from unittest import mock - import pytest -from testfixtures import LogCapture -from scrapy import signals -from scrapy.crawler import Crawler -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, XmlResponse -from scrapy.settings import Settings from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider -from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata -from tests.utils.decorators import inline_callbacks_test +from tests.utils.bases.spider import TestSpiderBase -class TestSpider: +class TestSpider(TestSpiderBase): spider_class = Spider - def test_base_spider(self): - spider = self.spider_class("example.com") - assert spider.name == "example.com" - assert spider.start_urls == [] - def test_spider_args(self): - """``__init__`` method arguments are assigned to spider attributes""" - spider = self.spider_class("example.com", foo="bar") - assert spider.foo == "bar" - - def test_spider_without_name(self): - """``__init__`` raises when the name is not provided.""" - msg = "must have a name" - with pytest.raises(ValueError, match=msg): - self.spider_class() - with pytest.raises(ValueError, match=msg): - self.spider_class(somearg="foo") - - def test_from_crawler_crawler_and_settings_population(self): - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - assert hasattr(spider, "crawler") - assert spider.crawler is crawler - assert hasattr(spider, "settings") - assert spider.settings is crawler.settings - - def test_from_crawler_init_call(self): - with mock.patch.object( - self.spider_class, "__init__", return_value=None - ) as mock_init: - self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar") - mock_init.assert_called_once_with("example.com", foo="bar") - - def test_closed_signal_call(self): - class TestSpider(self.spider_class): - closed_called = False - - def closed(self, reason): - self.closed_called = True - - crawler = get_crawler() - spider = TestSpider.from_crawler(crawler, "example.com") - crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider) - crawler.signals.send_catch_log( - signal=signals.spider_closed, spider=spider, reason=None - ) - assert spider.closed_called - - def test_update_settings(self): - spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} - self.spider_class.custom_settings = spider_settings - settings = Settings(project_settings, priority="project") - - self.spider_class.update_settings(settings) - assert settings.get("TEST1") == "spider" - assert settings.get("TEST2") == "spider" - assert settings.get("TEST3") == "project" - - @inline_callbacks_test - def test_settings_in_from_crawler(self): - spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = { - "TEST1": "project", - "TEST3": "project", - **get_reactor_settings(), - } - - class TestSpider(self.spider_class): - name = "test" - custom_settings = spider_settings - - @classmethod - def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): - spider = super().from_crawler(crawler, *args, **kwargs) - spider.settings.set("TEST1", "spider_instance", priority="spider") - return spider - - crawler = Crawler(TestSpider, project_settings) - assert crawler.settings.get("TEST1") == "spider" - assert crawler.settings.get("TEST2") == "spider" - assert crawler.settings.get("TEST3") == "project" - yield crawler.crawl() - assert crawler.settings.get("TEST1") == "spider_instance" - - def test_logger(self): - spider = self.spider_class("example.com") - with LogCapture() as lc: - spider.logger.info("test log msg") - lc.check(("example.com", "INFO", "test log msg")) - - record = lc.records[0] - assert "spider" in record.__dict__ - assert record.spider is spider - - def test_log(self): - spider = self.spider_class("example.com") - with ( - mock.patch("scrapy.spiders.Spider.logger") as mock_logger, - pytest.warns( - ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated" - ), - ): - spider.log("test log msg", "INFO") - mock_logger.log.assert_called_once_with("INFO", "test log msg") - - -class TestXMLFeedSpider(TestSpider): +class TestXMLFeedSpider(TestSpiderBase): spider_class = XMLFeedSpider def test_register_namespace(self): @@ -176,7 +61,7 @@ class TestXMLFeedSpider(TestSpider): ], iterator -class TestCSVFeedSpider(TestSpider): +class TestCSVFeedSpider(TestSpiderBase): spider_class = CSVFeedSpider def test_parse_rows(self): diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index 63010e195..f34f9add9 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -11,10 +11,10 @@ from scrapy.http import HtmlResponse, Request, TextResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule, Spider from scrapy.utils.test import get_crawler -from tests.test_spider import TestSpider +from tests.utils.bases.spider import TestSpiderBase -class TestCrawlSpider(TestSpider): +class TestCrawlSpider(TestSpiderBase): test_body = b"""Page title

Item 12

diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py index 0af99ab6d..c8cfd364e 100644 --- a/tests/test_spider_sitemap.py +++ b/tests/test_spider_sitemap.py @@ -14,11 +14,11 @@ from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlRespon from scrapy.spiders import SitemapSpider from scrapy.utils.test import get_crawler from tests import tests_datadir -from tests.test_spider import TestSpider +from tests.utils.bases.spider import TestSpiderBase from tests.utils.decorators import coroutine_test -class TestSitemapSpider(TestSpider): +class TestSitemapSpider(TestSpiderBase): spider_class = SitemapSpider BODY = b"SITEMAP" diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 67ce3a920..321139da3 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -5,7 +5,6 @@ import pytest from scrapy import Spider, signals from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler -from tests.test_spider_start import SLEEP_SECONDS from .utils import twisted_sleep from .utils.decorators import coroutine_test @@ -14,6 +13,8 @@ ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} ITEM_C = {"id": "c"} +SLEEP_SECONDS = 0.1 + class AsyncioSleepSpiderMiddleware: async def process_start(self, start): diff --git a/tests/utils/bases/__init__.py b/tests/utils/bases/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/utils/base_commands.py b/tests/utils/bases/commands.py similarity index 100% rename from tests/utils/base_commands.py rename to tests/utils/bases/commands.py diff --git a/tests/test_downloader_handlers_http_base.py b/tests/utils/bases/download_handlers_http.py similarity index 99% rename from tests/test_downloader_handlers_http_base.py rename to tests/utils/bases/download_handlers_http.py index 29e1d9757..e44f9bcb8 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/utils/bases/download_handlers_http.py @@ -1,5 +1,3 @@ -"""Base classes for HTTP download handler tests.""" - from __future__ import annotations import gzip diff --git a/tests/utils/bases/engine.py b/tests/utils/bases/engine.py new file mode 100644 index 000000000..1cabbb6ef --- /dev/null +++ b/tests/utils/bases/engine.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from itemadapter import ItemAdapter + +from scrapy import signals +from tests import get_testdata + +if TYPE_CHECKING: + from tests.utils.engine import CrawlerRun + + +class TestEngineBase: + @staticmethod + def _assert_visited_urls(run: CrawlerRun) -> None: + must_be_visited = [ + "/static/", + "/redirect", + "/redirected", + "/static/item1.html", + "/static/item2.html", + "/static/item999.html", + ] + urls_visited = {rp[0].url for rp in run.respplug} + urls_expected = {run.geturl(p) for p in must_be_visited} + assert urls_expected <= urls_visited, ( + f"URLs not visited: {list(urls_expected - urls_visited)}" + ) + + @staticmethod + def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None: + assert len(run.reqplug) == count + + paths_expected = [ + "/static/item999.html", + "/static/item2.html", + "/static/item1.html", + ] + + urls_requested = {rq[0].url for rq in run.reqplug} + urls_expected = {run.geturl(p) for p in paths_expected} + assert urls_expected <= urls_requested + scheduled_requests_count = len(run.reqplug) + dropped_requests_count = len(run.reqdropped) + responses_count = len(run.respplug) + assert scheduled_requests_count == dropped_requests_count + responses_count + assert len(run.reqreached) == responses_count + + @staticmethod + def _assert_dropped_requests(run: CrawlerRun) -> None: + assert len(run.reqdropped) == 1 + + @staticmethod + def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None: + # response tests + assert len(run.respplug) == count + assert len(run.reqreached) == count + + for response, _ in run.respplug: + if run.getpath(response.url) == "/static/item999.html": + assert response.status == 404 + if run.getpath(response.url) == "/redirect": + assert response.status == 302 + + @staticmethod + def _assert_items_error(run: CrawlerRun) -> None: + assert len(run.itemerror) == 2 + for item, response, spider, failure in run.itemerror: + assert failure.value.__class__ is ZeroDivisionError + assert spider == run.crawler.spider + + assert item["url"] == response.url + if "item1.html" in item["url"]: + assert item["name"] == "Item 1 name" + assert item["price"] == "100" + if "item2.html" in item["url"]: + assert item["name"] == "Item 2 name" + assert item["price"] == "200" + + @staticmethod + def _assert_scraped_items(run: CrawlerRun) -> None: + assert len(run.itemresp) == 2 + for item_, response in run.itemresp: + item = ItemAdapter(item_) + assert item["url"] == response.url + if "item1.html" in item["url"]: + assert item["name"] == "Item 1 name" + assert item["price"] == "100" + if "item2.html" in item["url"]: + assert item["name"] == "Item 2 name" + assert item["price"] == "200" + + @staticmethod + def _assert_headers_received(run: CrawlerRun) -> None: + for headers in run.headers.values(): + assert b"Server" in headers + assert headers[b"Server"] + assert b"TwistedWeb" in headers[b"Server"] + assert b"Date" in headers + assert b"Content-Type" in headers + + @staticmethod + def _assert_bytes_received(run: CrawlerRun) -> None: + assert len(run.bytes) == 9 + for request, data in run.bytes.items(): + joined_data = b"".join(data) + if run.getpath(request.url) == "/static/": + assert joined_data == get_testdata("test_site", "index.html") + elif run.getpath(request.url) == "/static/item1.html": + assert joined_data == get_testdata("test_site", "item1.html") + elif run.getpath(request.url) == "/static/item2.html": + assert joined_data == get_testdata("test_site", "item2.html") + elif run.getpath(request.url) == "/redirected": + assert joined_data == b"Redirected here" + elif run.getpath(request.url) == "/redirect": + assert ( + joined_data == b"\n\n" + b" \n" + b' \n' + b" \n" + b' \n' + b' click here\n' + b" \n" + b"\n" + ) + elif run.getpath(request.url) == "/static/item999.html": + assert ( + joined_data == b"\n\n" + b" 404 - No Such Resource\n" + b" \n" + b"

No Such Resource

\n" + b"

File not found.

\n" + b" \n" + b"\n" + ) + elif run.getpath(request.url) == "/numbers": + # signal was fired multiple times + assert len(data) > 1 + # bytes were received in order + numbers = [str(x).encode("utf8") for x in range(2**18)] + assert joined_data == b"".join(numbers) + + @staticmethod + def _assert_signals_caught(run: CrawlerRun) -> None: + assert signals.engine_started in run.signals_caught + assert signals.engine_stopped in run.signals_caught + assert signals.spider_opened in run.signals_caught + assert signals.spider_idle in run.signals_caught + assert signals.spider_closed in run.signals_caught + assert signals.headers_received in run.signals_caught + + assert {"spider": run.crawler.spider} == run.signals_caught[ + signals.spider_opened + ] + assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle] + assert { + "spider": run.crawler.spider, + "reason": "finished", + } == run.signals_caught[signals.spider_closed] diff --git a/tests/utils/bases/feedexport.py b/tests/utils/bases/feedexport.py new file mode 100644 index 000000000..f03ac1fd4 --- /dev/null +++ b/tests/utils/bases/feedexport.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import random +import shutil +import tempfile +from abc import ABC, abstractmethod +from pathlib import Path +from string import ascii_letters, digits +from typing import IO, TYPE_CHECKING, Any + +import scrapy +from scrapy import Spider +from tests.mockserver.http import MockServer + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + + +class TestFeedExportBase(ABC): + mockserver: MockServer + + def _random_temp_filename(self, inter_dir="") -> Path: + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = "".join(chars) + return Path(self.temp_dir, inter_dir, filename) + + @classmethod + def setup_class(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() # pylint: disable=unnecessary-dunder-call + + @classmethod + def teardown_class(cls): + cls.mockserver.__exit__(None, None, None) + + def setup_method(self): + self.temp_dir = tempfile.mkdtemp() + + def teardown_method(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + async def exported_data( + self, items: Iterable[Any], settings: dict[str, Any] + ) -> dict[str, Any]: + """ + Return exported data which a spider yielding ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = "testspider" + + def parse(self, response): + yield from items + + return await self.run_and_export(TestSpider, settings) + + async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]: + """ + Return exported data which a spider yielding no ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = "testspider" + + def parse(self, response): + pass + + return await self.run_and_export(TestSpider, settings) + + async def assertExported( + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + await self.assertExportedCsv(items, header, rows, settings) + await self.assertExportedJsonLines(items, rows, settings) + await self.assertExportedXml(items, rows, settings) + await self.assertExportedPickle(items, rows, settings) + await self.assertExportedMarshal(items, rows, settings) + await self.assertExportedMultiple(items, rows, settings) + + async def assertExportedCsv( # noqa: B027 + self, + items: Iterable[Any], + header: Iterable[str], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedJsonLines( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedXml( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMultiple( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedPickle( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + async def assertExportedMarshal( # noqa: B027 + self, + items: Iterable[Any], + rows: Iterable[dict[str, Any]], + settings: dict[str, Any] | None = None, + ) -> None: + pass + + @abstractmethod + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, Any]: + pass + + def _load_until_eof( + self, data: bytes, load_func: Callable[[IO[bytes]], Any] + ) -> list[Any]: + result: list[Any] = [] + with tempfile.TemporaryFile() as temp: + temp.write(data) + temp.seek(0) + while True: + try: + result.append(load_func(temp)) + except EOFError: + break + return result diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py new file mode 100644 index 000000000..c255b5e4c --- /dev/null +++ b/tests/utils/bases/http_request.py @@ -0,0 +1,490 @@ +import warnings +from abc import ABC, abstractmethod +from typing import Any + +import pytest + +from scrapy.http import Headers, Request +from scrapy.http.request import NO_CALLBACK + + +class TestRequestBase(ABC): + default_method = "GET" + default_headers: dict[bytes, list[bytes]] = {} + default_meta: dict[str, Any] = {} + + @property + @abstractmethod + def request_class(self) -> type[Request]: + raise NotImplementedError + + def test_init(self): + # Request requires url in the __init__ method + with pytest.raises(TypeError): + self.request_class() + + # url argument must be basestring + with pytest.raises(TypeError): + self.request_class(123) + + # priority argument must be an integer + with pytest.raises(TypeError, match="Request priority not an integer"): + self.request_class("http://www.example.com", priority="1") + + r = self.request_class("http://www.example.com") + assert isinstance(r.url, str) + assert r.url == "http://www.example.com" + assert r.method == self.default_method + + assert isinstance(r.headers, Headers) + assert r.headers == self.default_headers + assert r.meta == self.default_meta + + meta = {"lala": "lolo"} + headers = {b"caca": b"coco"} + r = self.request_class( + "http://www.example.com", meta=meta, headers=headers, body="a body" + ) + + assert r.meta is not meta + assert r.meta == meta + assert r.headers is not headers + assert r.headers[b"caca"] == b"coco" + + def test_url_scheme(self): + # This test passes by not raising any (ValueError) exception + self.request_class("http://example.org") + self.request_class("https://example.org") + self.request_class("s3://example.org") + self.request_class("ftp://example.org") + self.request_class("about:config") + self.request_class("data:,Hello%2C%20World!") + + def test_url_no_scheme(self): + msg = "Missing scheme in request url:" + with pytest.raises(ValueError, match=msg): + self.request_class("foo") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo/") + with pytest.raises(ValueError, match=msg): + self.request_class("/foo:bar") + + def test_headers(self): + # Different ways of setting headers attribute + url = "http://www.scrapy.org" + headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"} + r = self.request_class(url=url, headers=headers) + p = self.request_class(url=url, headers=r.headers) + + assert r.headers == p.headers + assert r.headers is not headers + assert p.headers is not r.headers + + # headers must not be unicode + h = Headers({"key1": "val1", "key2": "val2"}) + h["newkey"] = "newval" + for k, v in h.items(): + assert isinstance(k, bytes) + for s in v: + assert isinstance(s, bytes) + + def test_eq(self): + url = "http://www.scrapy.org" + r1 = self.request_class(url=url) + r2 = self.request_class(url=url) + assert r1 != r2 + + set_ = set() + set_.add(r1) + set_.add(r2) + assert len(set_) == 2 + + def test_url(self): + r = self.request_class(url="http://www.scrapy.org/path") + assert r.url == "http://www.scrapy.org/path" + + def test_url_quoting(self): + r = self.request_class(url="http://www.scrapy.org/blank%20space") + assert r.url == "http://www.scrapy.org/blank%20space" + r = self.request_class(url="http://www.scrapy.org/blank space") + assert r.url == "http://www.scrapy.org/blank%20space" + + def test_url_encoding(self): + r = self.request_class(url="http://www.scrapy.org/price/£") + assert r.url == "http://www.scrapy.org/price/%C2%A3" + + def test_url_encoding_other(self): + # encoding affects only query part of URI, not path + # path part should always be UTF-8 encoded before percent-escaping + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") + assert r.url == "http://www.scrapy.org/price/%C2%A3" + + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") + assert r.url == "http://www.scrapy.org/price/%C2%A3" + + def test_url_encoding_query(self): + r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") + assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" + + # should be same as above + r2 = self.request_class( + url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" + ) + assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5" + + def test_url_encoding_query_latin1(self): + # encoding is used for encoding query-string before percent-escaping; + # path is still UTF-8 encoded before percent-escaping + r3 = self.request_class( + url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" + ) + assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3" + + def test_url_encoding_nonutf8_untouched(self): + # percent-escaping sequences that do not match valid UTF-8 sequences + # should be kept untouched (just upper-cased perhaps) + # + # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2 + # + # "Conversions from URIs to IRIs MUST NOT use any character encoding + # other than UTF-8 in steps 3 and 4, even if it might be possible to + # guess from the context that another character encoding than UTF-8 was + # used in the URI. For example, the URI + # "http://www.example.org/r%E9sum%E9.html" might with some guessing be + # interpreted to contain two e-acute characters encoded as iso-8859-1. + # It must not be converted to an IRI containing these e-acute + # characters. Otherwise, in the future the IRI will be mapped to + # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different + # URI from "http://www.example.org/r%E9sum%E9.html". + r1 = self.request_class(url="http://www.scrapy.org/price/%a3") + assert r1.url == "http://www.scrapy.org/price/%a3" + + r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" + + r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") + assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3" + + r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") + assert r4.url == "http://www.example.org/r%E9sum%E9.html" + + def test_url_verbatim(self): + r = self.request_class( + url="http://www.scrapy.org/price/£", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/price/£" + + r = self.request_class( + url="http://www.scrapy.org/blank space", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/blank space" + + def test_body(self): + r1 = self.request_class(url="http://www.example.com/") + assert r1.body == b"" + + r2 = self.request_class(url="http://www.example.com/", body=b"") + assert isinstance(r2.body, bytes) + assert r2.encoding == "utf-8" # default encoding + + r3 = self.request_class( + url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" + ) + assert isinstance(r3.body, bytes) + assert r3.body == b"Price: \xc2\xa3100" + + r4 = self.request_class( + url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" + ) + assert isinstance(r4.body, bytes) + assert r4.body == b"Price: \xa3100" + + def test_copy(self): + """Test Request copy""" + + def somecallback(): + pass + + r1 = self.request_class( + "http://www.example.com", + flags=["f1", "f2"], + callback=somecallback, + errback=somecallback, + ) + r1.meta["foo"] = "bar" + r1.cb_kwargs["key"] = "value" + r2 = r1.copy() + + # make sure callbaclks are copied + assert r1.callback is somecallback + assert r1.errback is somecallback + assert r2.callback is r1.callback + assert r2.errback is r1.errback + + # make sure flags list is shallow copied + assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" + assert r1.flags == r2.flags + + # make sure cb_kwargs dict is shallow copied + assert r1.cb_kwargs is not r2.cb_kwargs, ( + "cb_kwargs must be a shallow copy, not identical" + ) + assert r1.cb_kwargs == r2.cb_kwargs + + # make sure meta dict is shallow copied + assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical" + assert r1.meta == r2.meta + + # make sure headers attribute is shallow copied + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) + assert r1.headers == r2.headers + assert r1.encoding == r2.encoding + assert r1.dont_filter == r2.dont_filter + + # Request.body can be identical since it's an immutable object (str) + + def test_copy_inherited_classes(self): + """Test Request children copies preserve their class""" + + class CustomRequest(self.request_class): + pass + + r1 = CustomRequest("http://www.example.com") + r2 = r1.copy() + + assert isinstance(r2, CustomRequest) + + def test_replace(self): + """Test Request.replace() method""" + r1 = self.request_class("http://www.example.com", method="GET") + hdrs = Headers(r1.headers) + hdrs[b"key"] = b"value" + r2 = r1.replace(method="POST", body="New body", headers=hdrs) + assert r1.url == r2.url + assert (r1.method, r2.method) == ("GET", "POST") + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == (self.default_headers, hdrs) + + # Empty attributes (which may fail if not compared properly) + r3 = self.request_class( + "http://www.example.com", meta={"a": 1}, dont_filter=True + ) + r4 = r3.replace( + url="http://www.example.com/2", body=b"", meta={}, dont_filter=False + ) + assert r4.url == "http://www.example.com/2" + assert r4.body == b"" + assert r4.meta == {} + assert r4.dont_filter is False + + # the cls argument allows changing the resulting class + custom_request_cls = type("CustomRequest", (self.request_class,), {}) + r5 = r1.replace(cls=custom_request_cls) + assert isinstance(r5, custom_request_cls) + assert r5.url == r1.url + + def test_method_always_str(self): + r = self.request_class("http://www.example.com", method="POST") + assert isinstance(r.method, str) + + def test_immutable_attributes(self): + r = self.request_class("http://example.com") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" + + def test_callback_and_errback(self): + def a_function(): + pass + + r1 = self.request_class("http://example.com") + assert r1.callback is None + assert r1.errback is None + + r2 = self.request_class("http://example.com", callback=a_function) + assert r2.callback is a_function + assert r2.errback is None + + r3 = self.request_class("http://example.com", errback=a_function) + assert r3.callback is None + assert r3.errback is a_function + + r4 = self.request_class( + url="http://example.com", + callback=a_function, + errback=a_function, + ) + assert r4.callback is a_function + assert r4.errback is a_function + + r5 = self.request_class( + url="http://example.com", + callback=NO_CALLBACK, + errback=NO_CALLBACK, + ) + assert r5.callback is NO_CALLBACK + assert r5.errback is NO_CALLBACK + + def test_callback_and_errback_type(self): + with pytest.raises(TypeError): + self.request_class("http://example.com", callback="a_function") + with pytest.raises(TypeError): + self.request_class("http://example.com", errback="a_function") + with pytest.raises(TypeError): + self.request_class( + url="http://example.com", + callback="a_function", + errback="a_function", + ) + + def test_setters(self): + request = self.request_class("http://example.com") + + request.flags = ["f1"] + assert request.flags == ["f1"] + + request.cookies = {"sid": "1"} + assert request.cookies == {"sid": "1"} + + headers = Headers({b"X-Test": b"1"}) + request.headers = headers + assert request._headers is headers + request.headers = {b"A": b"b"} + assert isinstance(request.headers, Headers) + assert request._headers[b"A"] == b"b" + + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + request = self.request_class("http://example.com") + + assert request._flags is None + assert request.flags == [] + assert request.flags is request.flags + assert request._flags == [] + original_flags = request.flags + request.flags = None + assert request._flags is None + assert request.flags == [] + assert request.flags is not original_flags + + assert request._cookies is None + assert request.cookies == {} + assert request.cookies is request.cookies + assert request._cookies == {} + original_cookies = request.cookies + request.cookies = None + assert request._cookies is None + assert request.cookies == {} + assert request.cookies is not original_cookies + + if self.default_headers: + assert request._headers == self.default_headers + assert request._headers is not self.default_headers + assert request.headers == self.default_headers + else: + assert request._headers is None + assert request.headers == {} + assert request.headers is request.headers + assert isinstance(request.headers, Headers) + assert isinstance(request._headers, Headers) + original_headers = request.headers + request.headers = None + assert request._headers is None + assert request.headers == {} + assert request._headers == {} + assert request.headers is not original_headers + + def test_no_callback(self): + with pytest.raises(RuntimeError): + NO_CALLBACK() + + def test_from_curl(self): + # Note: more curated tests regarding curl conversion are in + # `test_utils_curl.py` + curl_command = ( + "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique" + "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique" + "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H" + " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q" + "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use" + "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM" + "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S" + "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode" + "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=" + "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0" + "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee" + "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith" + "%40example.org&size=small&topping=cheese&topping=onion&delivery=1" + "2%3A15&comments=' --compressed" + ) + r = self.request_class.from_curl(curl_command) + assert r.method == "POST" + assert r.url == "http://httpbin.org/post" + assert ( + r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40" + b"example.org&size=small&topping=cheese&topping=onion" + b"&delivery=12%3A15&comments=" + ) + assert r.cookies == { + "_gauges_unique_year": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + } + assert r.headers == { + b"Origin": [b"http://httpbin.org"], + b"Accept-Encoding": [b"gzip, deflate"], + b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], + b"Upgrade-Insecure-Requests": [b"1"], + b"User-Agent": [ + b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." + b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" + b".75 Chrome/62.0.3202.75 Safari/537.36" + ], + b"Content-Type": [b"application /x-www-form-urlencoded"], + b"Accept": [ + b"text/html,application/xhtml+xml,application/xml;q=0." + b"9,image/webp,image/apng,*/*;q=0.8" + ], + b"Cache-Control": [b"max-age=0"], + b"Referer": [b"http://httpbin.org/forms/post"], + b"Connection": [b"keep-alive"], + } + + def test_from_curl_with_kwargs(self): + r = self.request_class.from_curl( + 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} + ) + assert r.method == "POST" + assert r.meta == {"key": "value"} + + def test_from_curl_ignore_unknown_options(self): + # By default: it works and ignores the unknown options: --foo and -z + with warnings.catch_warnings(): # avoid warning when executing tests + warnings.filterwarnings( + "ignore", category=UserWarning, message="Unrecognized options:" + ) + r = self.request_class.from_curl( + 'curl -X DELETE "http://example.org" --foo -z', + ) + assert r.method == "DELETE" + + # If `ignore_unknown_options` is set to `False` it raises an error with + # the unknown options: --foo and -z + with pytest.raises(ValueError, match="Unrecognized options:"): + self.request_class.from_curl( + 'curl -X PATCH "http://example.org" --foo -z', + ignore_unknown_options=False, + ) diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py new file mode 100644 index 000000000..2fbf6527a --- /dev/null +++ b/tests/utils/bases/http_response.py @@ -0,0 +1,417 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +import pytest +from w3lib.encoding import resolve_encoding + +from scrapy.exceptions import NotSupported +from scrapy.http import Headers, Request, Response +from scrapy.link import Link +from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS +from tests import get_testdata + +if TYPE_CHECKING: + from collections.abc import Iterable + + +class TestResponseBase(ABC): + @property + @abstractmethod + def response_class(self) -> type[Response]: + raise NotImplementedError + + def test_init(self): + # Response requires url in the constructor + with pytest.raises(TypeError): + self.response_class() + assert isinstance( + self.response_class("http://example.com/"), self.response_class + ) + with pytest.raises(TypeError): + self.response_class(b"http://example.com") + with pytest.raises(TypeError): + self.response_class(url="http://example.com", body={}) + # body can be str or None + assert isinstance( + self.response_class("http://example.com/", body=b""), + self.response_class, + ) + assert isinstance( + self.response_class("http://example.com/", body=b"body"), + self.response_class, + ) + # test presence of all optional parameters + assert isinstance( + self.response_class( + "http://example.com/", body=b"", headers={}, status=200 + ), + self.response_class, + ) + + r = self.response_class("http://www.example.com") + assert isinstance(r.url, str) + assert r.url == "http://www.example.com" + assert r.status == 200 + + assert isinstance(r.headers, Headers) + assert not r.headers + + headers = {"foo": "bar"} + body = b"a body" + r = self.response_class("http://www.example.com", headers=headers, body=body) + + assert r.headers is not headers + assert r.headers[b"foo"] == b"bar" + + r = self.response_class("http://www.example.com", status=301) + assert r.status == 301 + r = self.response_class("http://www.example.com", status="301") + assert r.status == 301 + with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): + self.response_class("http://example.com", status="lala200") + + def test_copy(self): + """Test Response copy""" + + r1 = self.response_class("http://www.example.com", body=b"Some body") + r1.flags.append("cached") + r2 = r1.copy() + + assert r1.status == r2.status + assert r1.body == r2.body + + # make sure flags list is shallow copied + assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" + assert r1.flags == r2.flags + + # make sure headers attribute is shallow copied + assert r1.headers is not r2.headers, ( + "headers must be a shallow copy, not identical" + ) + assert r1.headers == r2.headers + + def test_copy_meta(self): + req = Request("http://www.example.com") + req.meta["foo"] = "bar" + r1 = self.response_class( + "http://www.example.com", body=b"Some body", request=req + ) + assert r1.meta is req.meta + + def test_copy_cb_kwargs(self): + req = Request("http://www.example.com") + req.cb_kwargs["foo"] = "bar" + r1 = self.response_class( + "http://www.example.com", body=b"Some body", request=req + ) + assert r1.cb_kwargs is req.cb_kwargs + + def test_unavailable_meta(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with pytest.raises(AttributeError, match=r"Response\.meta not available"): + r1.meta # pylint: disable=pointless-statement + + def test_unavailable_cb_kwargs(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"): + r1.cb_kwargs # pylint: disable=pointless-statement + + def test_copy_inherited_classes(self): + """Test Response children copies preserve their class""" + + class CustomResponse(self.response_class): + pass + + r1 = CustomResponse("http://www.example.com") + r2 = r1.copy() + + assert isinstance(r2, CustomResponse) + + def test_replace(self): + """Test Response.replace() method""" + hdrs = Headers({"key": "value"}) + r1 = self.response_class("http://www.example.com") + r2 = r1.replace(status=301, body=b"New body", headers=hdrs) + assert r1.body == b"" + assert r1.url == r2.url + assert (r1.status, r2.status) == (200, 301) + assert (r1.body, r2.body) == (b"", b"New body") + assert (r1.headers, r2.headers) == ({}, hdrs) + + # Empty attributes (which may fail if not compared properly) + r3 = self.response_class("http://www.example.com", flags=["cached"]) + r4 = r3.replace(body=b"", flags=[]) + assert r4.body == b"" + assert not r4.flags + + def _assert_response_values(self, response, encoding, body): + if isinstance(body, str): + body_unicode = body + body_bytes = body.encode(encoding) + else: + body_unicode = body.decode(encoding) + body_bytes = body + + assert isinstance(response.body, bytes) + assert isinstance(response.text, str) + self._assert_response_encoding(response, encoding) + assert response.body == body_bytes + assert response.text == body_unicode + + def _assert_response_encoding(self, response, encoding): + assert response.encoding == resolve_encoding(encoding) + + def test_immutable_attributes(self): + r = self.response_class("http://example.com") + with pytest.raises(AttributeError): + r.url = "http://example2.com" + with pytest.raises(AttributeError): + r.body = "xxx" + + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + response = self.response_class("http://example.com") + + response.request = Request("http://example.com") + + assert response._flags is None + assert response.flags == [] + assert response.flags is response.flags + assert response._flags == [] + original_flags = response.flags + response.flags = None + assert response._flags is None + assert response.flags == [] + assert response.flags is not original_flags + + assert response._headers is None + assert response.headers == {} + assert response.headers is response.headers + assert isinstance(response.headers, Headers) + assert isinstance(response._headers, Headers) + original_headers = response.headers + response.headers = None + assert response._headers is None + assert response.headers == {} + assert response._headers == {} + assert response.headers is not original_headers + + def test_setters(self): + response = self.response_class("http://example.com") + + response.flags = ["f1"] + assert response.flags == ["f1"] + + headers = Headers({b"X-Test": b"1"}) + response.headers = headers + assert response._headers is headers + response.headers = {b"A": b"b"} + assert isinstance(response.headers, Headers) + assert response._headers[b"A"] == b"b" + + def test_urljoin(self): + """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" + joined = self.response_class("http://www.example.com").urljoin("/test") + absolute = "http://www.example.com/test" + assert joined == absolute + + def test_shortcut_attributes(self): + r = self.response_class("http://example.com", body=b"hello") + if self.response_class == Response: + msg = "Response content isn't text" + with pytest.raises(AttributeError, match=msg): + r.text # pylint: disable=pointless-statement + with pytest.raises(NotSupported, match=msg): + r.css("body") + with pytest.raises(NotSupported, match=msg): + r.xpath("//body") + with pytest.raises(NotSupported, match=msg): + r.jmespath("body") + else: + r.text # pylint: disable=pointless-statement + r.css("body") + r.xpath("//body") + + # Response.follow + + def test_follow_url_absolute(self): + self._assert_followed_url("http://foo.example.com", "http://foo.example.com") + + def test_follow_url_relative(self): + self._assert_followed_url("foo", "http://example.com/foo") + + def test_follow_link(self): + self._assert_followed_url( + Link("http://example.com/foo"), "http://example.com/foo" + ) + + def test_follow_None_url(self): + r = self.response_class("http://example.com") + with pytest.raises(ValueError, match="url can't be None"): + r.follow(None) + + def test_follow_None_encoding(self): + r = self.response_class("http://example.com") + with pytest.raises(ValueError, match="encoding can't be None"): + r.follow("foo", encoding=None) + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_whitespace_url(self): + self._assert_followed_url("foo ", "http://example.com/foo") + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_whitespace_link(self): + self._assert_followed_url( + Link("http://example.com/foo "), "http://example.com/foo" + ) + + def test_follow_flags(self): + res = self.response_class("http://example.com/") + fol = res.follow("http://example.com/", flags=["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] + + # Response.follow_all + + def test_follow_all_absolute(self): + url_list = [ + "http://example.org", + "http://www.example.org", + "http://example.com", + "http://www.example.com", + ] + self._assert_followed_all_urls(url_list, url_list) + + def test_follow_all_relative(self): + relative = ["foo", "bar", "foo/bar", "bar/foo"] + absolute = [ + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", + ] + self._assert_followed_all_urls(relative, absolute) + + def test_follow_all_links(self): + absolute = [ + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", + ] + links = map(Link, absolute) + self._assert_followed_all_urls(links, absolute) + + def test_follow_all_empty(self): + r = self.response_class("http://example.com") + assert not list(r.follow_all([])) + + def test_follow_all_invalid(self): + r = self.response_class("http://example.com") + if self.response_class == Response: + with pytest.raises(TypeError): + list(r.follow_all(urls=None)) + with pytest.raises(TypeError): + list(r.follow_all(urls=12345)) + with pytest.raises(ValueError, match="url can't be None"): + list(r.follow_all(urls=[None])) + else: + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): + list(r.follow_all(urls=None)) + with pytest.raises(TypeError): + list(r.follow_all(urls=12345)) + with pytest.raises(ValueError, match="url can't be None"): + list(r.follow_all(urls=[None])) + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_all_whitespace(self): + relative = ["foo ", "bar ", "foo/bar ", "bar/foo "] + absolute = [ + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", + ] + self._assert_followed_all_urls(relative, absolute) + + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) + def test_follow_all_whitespace_links(self): + absolute = [ + "http://example.com/foo ", + "http://example.com/bar ", + "http://example.com/foo/bar ", + "http://example.com/bar/foo ", + ] + links = [Link(u) for u in absolute] + expected = [u.strip() for u in absolute] + self._assert_followed_all_urls(links, expected) + + def test_follow_all_flags(self): + re = self.response_class("http://www.example.com/") + urls = [ + "http://www.example.com/", + "http://www.example.com/2", + "http://www.example.com/foo", + ] + fol = re.follow_all(urls, flags=["cached", "allowed"]) + for req in fol: + assert req.flags == ["cached", "allowed"] + + def _assert_followed_url( + self, + follow_obj: str | Link, + target_url: str, + response: Response | None = None, + encoding: str | None = None, + ) -> None: + if response is None: + response = self._links_response() + req = response.follow(follow_obj) + assert req.url == target_url + if encoding is not None: + assert req.encoding == encoding + + def _assert_followed_all_urls( + self, + follow_obj: Iterable[str | Link], + target_urls: Iterable[str], + response: Response | None = None, + ) -> None: + if response is None: + response = self._links_response() + followed = response.follow_all(follow_obj) + for req, target in zip(followed, target_urls, strict=True): + assert req.url == target + + def _links_response(self) -> Response: + body = get_testdata("link_extractor", "linkextractor.html") + return self.response_class("http://example.com/index", body=body) + + def _links_response_no_href(self) -> Response: + body = get_testdata("link_extractor", "linkextractor_no_href.html") + return self.response_class("http://example.com/index", body=body) diff --git a/tests/utils/bases/redirect.py b/tests/utils/bases/redirect.py new file mode 100644 index 000000000..e77a30533 --- /dev/null +++ b/tests/utils/bases/redirect.py @@ -0,0 +1,988 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any + +import pytest + +from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.http import Request, Response +from scrapy.utils.misc import set_environ +from scrapy.utils.test import get_crawler + + +class TestRedirectBase(ABC): + mwcls: type[Any] + mw: Any + reason: int | str + + @abstractmethod + def get_response( + self, request: Request, location: str, status: int = 302 + ) -> Response: + raise NotImplementedError + + def test_priority_adjust(self): + req = Request("http://a.example") + rsp = self.get_response(req, "http://a.example/redirected") + req2 = self.mw.process_response(req, rsp) + assert req2.priority > req.priority + + def test_dont_redirect(self): + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" + req = Request(url, meta={"dont_redirect": True}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Response) + assert r is rsp + + # Test that it redirects when dont_redirect is False + req = Request(url, meta={"dont_redirect": False}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Request) + + def test_post(self): + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request( + url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = self.get_response(req, url2) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) + assert not req2.body, f"Redirected body must be empty, not '{req2.body!r}'" + + def test_max_redirect_times(self): + self.mw.max_redirect_times = 1 + req = Request("http://a.example/302") + rsp = self.get_response(req, "/redirected") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + assert "redirect_times" in req.meta + assert req.meta["redirect_times"] == 1 + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_ttl(self): + self.mw.max_redirect_times = 100 + req = Request("http://a.example/302", meta={"redirect_ttl": 1}) + rsp = self.get_response(req, "/a") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_redirect_urls(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + + assert req2.url == "http://a.example/redirected" + assert req2.meta["redirect_urls"] == ["http://a.example/first"] + assert req3.url == "http://a.example/redirected2" + assert req3.meta["redirect_urls"] == [ + "http://a.example/first", + "http://a.example/redirected", + ] + + def test_redirect_reasons(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected1") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + assert req2.meta["redirect_reasons"] == [self.reason] + assert req3.meta["redirect_reasons"] == [self.reason, self.reason] + + def test_cross_origin_header_dropping(self): + safe_headers = {"A": "B"} + cookie_header = {"Cookie": "a=b"} + authorization_header = {"Authorization": "Bearer 123456"} + + original_request = Request( + "https://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers. + internal_response = self.get_response(original_request, "https://example.com/a") + internal_redirect_request = self.mw.process_response( + original_request, internal_response + ) + assert isinstance(internal_redirect_request, Request) + assert original_request.headers == internal_redirect_request.headers + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers also when the scheme is http. + http_request = Request( + "http://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + http_response = self.get_response(http_request, "http://example.com/a") + http_redirect_request = self.mw.process_response(http_request, http_response) + assert isinstance(http_redirect_request, Request) + assert http_request.headers == http_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_explicit_port_response = self.get_response( + original_request, "https://example.com:443/a" + ) + to_explicit_port_redirect_request = self.mw.process_response( + original_request, to_explicit_port_response + ) + assert isinstance(to_explicit_port_redirect_request, Request) + assert original_request.headers == to_explicit_port_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_implicit_port_response = self.get_response( + original_request, "https://example.com/a" + ) + to_implicit_port_redirect_request = self.mw.process_response( + original_request, to_implicit_port_response + ) + assert isinstance(to_implicit_port_redirect_request, Request) + assert original_request.headers == to_implicit_port_redirect_request.headers + + # A port change drops the Authorization header because the origin + # changes, but keeps the Cookie header because the domain remains the + # same. + different_port_response = self.get_response( + original_request, "https://example.com:8080/a" + ) + different_port_redirect_request = self.mw.process_response( + original_request, different_port_response + ) + assert isinstance(different_port_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == different_port_redirect_request.headers.to_unicode_dict() + + # A domain change drops both the Authorization and the Cookie header. + external_response = self.get_response(original_request, "https://example.org/a") + external_redirect_request = self.mw.process_response( + original_request, external_response + ) + assert isinstance(external_redirect_request, Request) + assert safe_headers == external_redirect_request.headers.to_unicode_dict() + + # A scheme upgrade (http → https) drops the Authorization header + # because the origin changes, but keeps the Cookie header because the + # domain remains the same. + upgrade_response = self.get_response(http_request, "https://example.com/a") + upgrade_redirect_request = self.mw.process_response( + http_request, upgrade_response + ) + assert isinstance(upgrade_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == upgrade_redirect_request.headers.to_unicode_dict() + + # A scheme downgrade (https → http) drops the Authorization header + # because the origin changes, and the Cookie header because its value + # cannot indicate whether the cookies were secure (HTTPS-only) or not. + # + # Note: If the Cookie header is set by the cookie management + # middleware, as recommended in the docs, the dropping of Cookie on + # scheme downgrade is not an issue, because the cookie management + # middleware will add again the Cookie header to the new request if + # appropriate. + downgrade_response = self.get_response(original_request, "http://example.com/a") + downgrade_redirect_request = self.mw.process_response( + original_request, downgrade_response + ) + assert isinstance(downgrade_redirect_request, Request) + assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() + + def test_meta_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_to_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_to_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_unproxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_proxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_proxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_unproxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.py new file mode 100644 index 000000000..f774fde07 --- /dev/null +++ b/tests/utils/bases/spider.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any +from unittest import mock + +import pytest +from testfixtures import LogCapture + +from scrapy import signals +from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.settings import Settings +from scrapy.utils.test import get_crawler, get_reactor_settings +from tests.utils.decorators import inline_callbacks_test + +if TYPE_CHECKING: + from scrapy.spiders import Spider + + +class TestSpiderBase(ABC): + @property + @abstractmethod + def spider_class(self) -> type[Spider]: + raise NotImplementedError + + def test_base_spider(self): + spider = self.spider_class("example.com") + assert spider.name == "example.com" + assert spider.start_urls == [] + + def test_spider_args(self): + """``__init__`` method arguments are assigned to spider attributes""" + spider = self.spider_class("example.com", foo="bar") + assert spider.foo == "bar" + + def test_spider_without_name(self): + """``__init__`` raises when the name is not provided.""" + msg = "must have a name" + with pytest.raises(ValueError, match=msg): + self.spider_class() + with pytest.raises(ValueError, match=msg): + self.spider_class(somearg="foo") + + def test_from_crawler_crawler_and_settings_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + assert hasattr(spider, "crawler") + assert spider.crawler is crawler + assert hasattr(spider, "settings") + assert spider.settings is crawler.settings + + def test_from_crawler_init_call(self): + with mock.patch.object( + self.spider_class, "__init__", return_value=None + ) as mock_init: + self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar") + mock_init.assert_called_once_with("example.com", foo="bar") + + def test_closed_signal_call(self): + class TestSpider(self.spider_class): + closed_called = False + + def closed(self, reason): + self.closed_called = True + + crawler = get_crawler() + spider = TestSpider.from_crawler(crawler, "example.com") + crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider) + crawler.signals.send_catch_log( + signal=signals.spider_closed, spider=spider, reason=None + ) + assert spider.closed_called + + def test_update_settings(self): + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = {"TEST1": "project", "TEST3": "project"} + self.spider_class.custom_settings = spider_settings + settings = Settings(project_settings, priority="project") + + self.spider_class.update_settings(settings) + assert settings.get("TEST1") == "spider" + assert settings.get("TEST2") == "spider" + assert settings.get("TEST3") == "project" + + @inline_callbacks_test + def test_settings_in_from_crawler(self): + spider_settings = {"TEST1": "spider", "TEST2": "spider"} + project_settings = { + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } + + class TestSpider(self.spider_class): + name = "test" + custom_settings = spider_settings + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("TEST1", "spider_instance", priority="spider") + return spider + + crawler = Crawler(TestSpider, project_settings) + assert crawler.settings.get("TEST1") == "spider" + assert crawler.settings.get("TEST2") == "spider" + assert crawler.settings.get("TEST3") == "project" + yield crawler.crawl() + assert crawler.settings.get("TEST1") == "spider_instance" + + def test_logger(self): + spider = self.spider_class("example.com") + with LogCapture() as lc: + spider.logger.info("test log msg") + lc.check(("example.com", "INFO", "test log msg")) + + record = lc.records[0] + assert "spider" in record.__dict__ + assert record.spider is spider + + def test_log(self): + spider = self.spider_class("example.com") + with ( + mock.patch("scrapy.spiders.Spider.logger") as mock_logger, + pytest.warns( + ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated" + ), + ): + spider.log("test log msg", "INFO") + mock_logger.log.assert_called_once_with("INFO", "test log msg") diff --git a/tests/utils/downloader.py b/tests/utils/downloader.py new file mode 100644 index 000000000..f64c02332 --- /dev/null +++ b/tests/utils/downloader.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, NamedTuple, cast + +from scrapy.core.downloader import Downloader +from scrapy.utils.httpobj import urlparse_cached + +if TYPE_CHECKING: + from scrapy.http import Request + + +class MockSlot(NamedTuple): + active: list[Any] + + +class MockDownloader: + def __init__(self) -> None: + self.slots: dict[str, MockSlot] = {} + + def get_slot_key(self, request: Request) -> str: + if Downloader.DOWNLOAD_SLOT in request.meta: + return cast("str", request.meta[Downloader.DOWNLOAD_SLOT]) + + return urlparse_cached(request).hostname or "" + + def increment(self, slot_key: str) -> None: + slot = self.slots.setdefault(slot_key, MockSlot(active=[])) + slot.active.append(1) + + def decrement(self, slot_key: str) -> None: + slot = self.slots[slot_key] + slot.active.pop() + + def close(self) -> None: + pass diff --git a/tests/utils/engine.py b/tests/utils/engine.py new file mode 100644 index 000000000..c193d39a9 --- /dev/null +++ b/tests/utils/engine.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import re +from collections import defaultdict +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any +from urllib.parse import urlparse + +import attr +from itemadapter import ItemAdapter +from pydispatch import dispatcher +from twisted.internet import defer + +from scrapy import signals +from scrapy.http import Headers, Request, Response +from scrapy.item import Field, Item +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import Spider +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.signal import disconnect_all +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from twisted.python.failure import Failure + + from tests.mockserver.http import MockServer + + +class MyItem(Item): + name = Field() + url = Field() + price = Field() + + +@attr.s +class AttrsItem: + name = attr.ib(default="") + url = attr.ib(default="") + price = attr.ib(default=0) + + +@dataclass +class DataClassItem: + name: str = "" + url: str = "" + price: int = 0 + + +class MySpider(Spider): + name = "scrapytest.org" + + itemurl_re = re.compile(r"item\d+.html") + name_re = re.compile(r"

(.*?)

", re.MULTILINE) + price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE) + + item_cls: type = MyItem + + def parse(self, response): + xlink = LinkExtractor() + itemre = re.compile(self.itemurl_re) + for link in xlink.extract_links(response): + if itemre.search(link.url): + yield Request(url=link.url, callback=self.parse_item) + + def parse_item(self, response): + adapter = ItemAdapter(self.item_cls()) + m = self.name_re.search(response.text) + if m: + adapter["name"] = m.group(1) + adapter["url"] = response.url + m = self.price_re.search(response.text) + if m: + adapter["price"] = m.group(1) + return adapter.item + + +class DictItemsSpider(MySpider): + item_cls = dict + + +class AttrsItemsSpider(MySpider): + item_cls = AttrsItem + + +class DataClassItemsSpider(MySpider): + item_cls = DataClassItem + + +class CrawlerRun: + """A class to run the crawler and keep track of events occurred""" + + def __init__(self, spider_class: type[Spider]): + self.respplug: list[tuple[Response, Spider]] = [] + self.reqplug: list[tuple[Request, Spider]] = [] + self.reqdropped: list[tuple[Request, Spider]] = [] + self.reqreached: list[tuple[Request, Spider]] = [] + self.itemerror: list[tuple[Any, Response, Spider, Failure]] = [] + self.itemresp: list[tuple[Any, Response]] = [] + self.headers: dict[Request, Headers] = {} + self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list) + self.signals_caught: dict[Any, dict[str, Any]] = {} + self.spider_class = spider_class + + async def run(self, mockserver: MockServer) -> None: + self.mockserver = mockserver + + start_urls = [ + self.geturl("/static/"), + self.geturl("/redirect"), + self.geturl("/redirect"), # duplicate + self.geturl("/numbers"), + ] + + for name, signal in vars(signals).items(): + if not name.startswith("_"): + dispatcher.connect(self.record_signal, signal) + + self.crawler = get_crawler(self.spider_class) + self.crawler.signals.connect(self.item_scraped, signals.item_scraped) + self.crawler.signals.connect(self.item_error, signals.item_error) + self.crawler.signals.connect(self.headers_received, signals.headers_received) + self.crawler.signals.connect(self.bytes_received, signals.bytes_received) + self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) + self.crawler.signals.connect(self.request_dropped, signals.request_dropped) + self.crawler.signals.connect( + self.request_reached, signals.request_reached_downloader + ) + self.crawler.signals.connect( + self.response_downloaded, signals.response_downloaded + ) + self.crawler.crawl(start_urls=start_urls) + + self.deferred: defer.Deferred[None] = defer.Deferred() + dispatcher.connect(self.stop, signals.engine_stopped) + await maybe_deferred_to_future(self.deferred) + + async def stop(self): + for name, signal in vars(signals).items(): + if not name.startswith("_"): + disconnect_all(signal) + self.deferred.callback(None) + await self.crawler.stop_async() + + def geturl(self, path: str) -> str: + return self.mockserver.url(path) + + def getpath(self, url: str) -> str: + u = urlparse(url) + return u.path + + def item_error( + self, item: Any, response: Response, spider: Spider, failure: Failure + ) -> None: + self.itemerror.append((item, response, spider, failure)) + + def item_scraped(self, item: Any, spider: Spider, response: Response) -> None: + self.itemresp.append((item, response)) + + def headers_received( + self, headers: Headers, body_length: int, request: Request, spider: Spider + ) -> None: + self.headers[request] = headers + + def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: + self.bytes[request].append(data) + + def request_scheduled(self, request: Request, spider: Spider) -> None: + self.reqplug.append((request, spider)) + + def request_reached(self, request: Request, spider: Spider) -> None: + self.reqreached.append((request, spider)) + + def request_dropped(self, request: Request, spider: Spider) -> None: + self.reqdropped.append((request, spider)) + + def response_downloaded(self, response: Response, spider: Spider) -> None: + self.respplug.append((response, spider)) + + def record_signal(self, *args: Any, **kwargs: Any) -> None: + """Record a signal and its parameters""" + signalargs = kwargs.copy() + sig = signalargs.pop("signal") + signalargs.pop("sender", None) + self.signals_caught[sig] = signalargs diff --git a/tests/utils/media_pipelines.py b/tests/utils/media_pipelines.py new file mode 100644 index 000000000..283c95940 --- /dev/null +++ b/tests/utils/media_pipelines.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from typing import Any + +from scrapy.http.request import NO_CALLBACK, Request + + +async def mocked_download_func(request: Request) -> Any: + assert request.callback is NO_CALLBACK + response = request.meta.get("response") + if callable(response): + response = await response() + if isinstance(response, Exception): + raise response + return response