RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove remaining cross-imports in test modules. (#7782)

This commit is contained in:
Andrey Rakhmatullin 2026-07-27 15:19:12 +05:00 committed by GitHub
parent 13be37e4b1
commit cec86f216e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
44 changed files with 2642 additions and 2536 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pyproject.toml
View File

@ -185,6 +185,9 @@ module = [
"tests.test_utils_misc.test_return_with_argument_inside_generator",
"tests.test_utils_python",
"tests.test_utils_request",
"tests.utils.bases.http_request",
"tests.utils.bases.http_response",
"tests.utils.bases.spider",
]
check_untyped_defs = false

8
tests/mockserver/http_base.py
View File

@ -20,6 +20,9 @@ if TYPE_CHECKING:
from twisted.web import resource
# typing.Self requires Python 3.11
from typing_extensions import Self
class BaseMockServer(ABC):
listen_http: bool = True
@ -39,13 +42,14 @@ class BaseMockServer(ABC):
self.http_port: int | None = None
self.https_port: int | None = None
def __enter__(self):
def __enter__(self) -> Self:
self.proc = Popen(
[sys.executable, "-u", "-m", self.module_name, *self.get_additional_args()],
stdout=PIPE,
env=get_script_run_env(),
text=True,
)
assert self.proc.stdout is not None
if self.listen_http:
http_address = self.proc.stdout.readline().strip()
http_parsed = urlparse(http_address)
@ -56,7 +60,7 @@ class BaseMockServer(ABC):
self.https_port = https_parsed.port
return self
def __exit__(self, exc_type, exc_value, traceback):
def __exit__(self, exc_type, exc_value, traceback) -> None:
if self.proc:
self.proc.kill()
self.proc.communicate()

2
tests/test_command_check.py
View File

@ -7,7 +7,7 @@ from unittest import TestCase
from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
from scrapy.commands.check import Command, TextTestResult
from tests.utils.base_commands import TestProjectBase
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import proc
if TYPE_CHECKING:

2
tests/test_command_crawl.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from tests.utils.base_commands import TestProjectBase
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import proc
if TYPE_CHECKING:

2
tests/test_command_genspider.py
View File

@ -6,7 +6,7 @@ from pathlib import Path
import pytest
from tests.utils.base_commands import TestProjectBase
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import call, proc, write_recording_editor

2
tests/test_command_parse.py
View File

@ -8,7 +8,7 @@ import pytest
from scrapy.commands import parse
from scrapy.settings import Settings
from tests.utils.base_commands import TestProjectBase
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import call, proc
if TYPE_CHECKING:

2
tests/test_commands.py
View File

@ -15,7 +15,7 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.reactor import _asyncio_reactor_path
from tests.utils.base_commands import TestProjectBase
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import call, proc, write_recording_editor
if TYPE_CHECKING:

2
tests/test_downloader_handler_httpx.py
View File

@ -15,7 +15,7 @@ from scrapy.core.downloader.handlers._httpx import (
HttpxDownloadHandler,
)
from scrapy.exceptions import DownloadFailedError
from tests.test_downloader_handlers_http_base import (
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
TestHttpProxyBase,
TestHttpsBase,

2
tests/test_downloader_handler_twisted_http11.py
View File

@ -11,7 +11,7 @@ from scrapy import Spider
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from tests.test_downloader_handlers_http_base import (
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
TestHttpProxyBase,
TestHttpsBase,

2
tests/test_downloader_handler_twisted_http2.py
View File

@ -13,7 +13,7 @@ from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import DownloadFailedError, NotConfigured
from scrapy.http import Request
from tests.test_downloader_handlers_http_base import (
from tests.utils.bases.download_handlers_http import (
TestHttpProxyBase,
TestHttpsBase,
TestHttpsCustomCiphersBase,

4
tests/test_downloadermiddleware_redirect.py
View File

@ -16,11 +16,11 @@ from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.test_downloadermiddleware_redirect_base import Base
from tests.utils.bases.redirect import TestRedirectBase
from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS
class TestRedirectMiddleware(Base.Test):
class TestRedirectMiddleware(TestRedirectBase):
mwcls = RedirectMiddleware
reason = 302

984
tests/test_downloadermiddleware_redirect_base.py
View File

@ -1,984 +0,0 @@
from __future__ import annotations
import pytest
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.http import Request, Response
from scrapy.utils.misc import set_environ
from scrapy.utils.test import get_crawler
class Base:
class Test:
def test_priority_adjust(self):
req = Request("http://a.example")
rsp = self.get_response(req, "http://a.example/redirected")
req2 = self.mw.process_response(req, rsp)
assert req2.priority > req.priority
def test_dont_redirect(self):
url = "http://www.example.com/301"
url2 = "http://www.example.com/redirected"
req = Request(url, meta={"dont_redirect": True})
rsp = self.get_response(req, url2)
r = self.mw.process_response(req, rsp)
assert isinstance(r, Response)
assert r is rsp
# Test that it redirects when dont_redirect is False
req = Request(url, meta={"dont_redirect": False})
rsp = self.get_response(req, url2)
r = self.mw.process_response(req, rsp)
assert isinstance(r, Request)
def test_post(self):
url = "http://www.example.com/302"
url2 = "http://www.example.com/redirected2"
req = Request(
url,
method="POST",
body="test",
headers={"Content-Type": "text/plain", "Content-length": "4"},
)
rsp = self.get_response(req, url2)
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.url == url2
assert req2.method == "GET"
assert "Content-Type" not in req2.headers, (
"Content-Type header must not be present in redirected request"
)
assert "Content-Length" not in req2.headers, (
"Content-Length header must not be present in redirected request"
)
assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"
def test_max_redirect_times(self):
self.mw.max_redirect_times = 1
req = Request("http://a.example/302")
rsp = self.get_response(req, "/redirected")
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert "redirect_times" in req.meta
assert req.meta["redirect_times"] == 1
with pytest.raises(IgnoreRequest):
self.mw.process_response(req, rsp)
def test_ttl(self):
self.mw.max_redirect_times = 100
req = Request("http://a.example/302", meta={"redirect_ttl": 1})
rsp = self.get_response(req, "/a")
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
with pytest.raises(IgnoreRequest):
self.mw.process_response(req, rsp)
def test_redirect_urls(self):
req1 = Request("http://a.example/first")
rsp1 = self.get_response(req1, "/redirected")
req2 = self.mw.process_response(req1, rsp1)
rsp2 = self.get_response(req2, "/redirected2")
req3 = self.mw.process_response(req2, rsp2)
assert req2.url == "http://a.example/redirected"
assert req2.meta["redirect_urls"] == ["http://a.example/first"]
assert req3.url == "http://a.example/redirected2"
assert req3.meta["redirect_urls"] == [
"http://a.example/first",
"http://a.example/redirected",
]
def test_redirect_reasons(self):
req1 = Request("http://a.example/first")
rsp1 = self.get_response(req1, "/redirected1")
req2 = self.mw.process_response(req1, rsp1)
rsp2 = self.get_response(req2, "/redirected2")
req3 = self.mw.process_response(req2, rsp2)
assert req2.meta["redirect_reasons"] == [self.reason]
assert req3.meta["redirect_reasons"] == [self.reason, self.reason]
def test_cross_origin_header_dropping(self):
safe_headers = {"A": "B"}
cookie_header = {"Cookie": "a=b"}
authorization_header = {"Authorization": "Bearer 123456"}
original_request = Request(
"https://example.com",
headers={**safe_headers, **cookie_header, **authorization_header},
)
# Redirects to the same origin (same scheme, same domain, same port)
# keep all headers.
internal_response = self.get_response(
original_request, "https://example.com/a"
)
internal_redirect_request = self.mw.process_response(
original_request, internal_response
)
assert isinstance(internal_redirect_request, Request)
assert original_request.headers == internal_redirect_request.headers
# Redirects to the same origin (same scheme, same domain, same port)
# keep all headers also when the scheme is http.
http_request = Request(
"http://example.com",
headers={**safe_headers, **cookie_header, **authorization_header},
)
http_response = self.get_response(http_request, "http://example.com/a")
http_redirect_request = self.mw.process_response(
http_request, http_response
)
assert isinstance(http_redirect_request, Request)
assert http_request.headers == http_redirect_request.headers
# For default ports, whether the port is explicit or implicit does not
# affect the outcome, it is still the same origin.
to_explicit_port_response = self.get_response(
original_request, "https://example.com:443/a"
)
to_explicit_port_redirect_request = self.mw.process_response(
original_request, to_explicit_port_response
)
assert isinstance(to_explicit_port_redirect_request, Request)
assert original_request.headers == to_explicit_port_redirect_request.headers
# For default ports, whether the port is explicit or implicit does not
# affect the outcome, it is still the same origin.
to_implicit_port_response = self.get_response(
original_request, "https://example.com/a"
)
to_implicit_port_redirect_request = self.mw.process_response(
original_request, to_implicit_port_response
)
assert isinstance(to_implicit_port_redirect_request, Request)
assert original_request.headers == to_implicit_port_redirect_request.headers
# A port change drops the Authorization header because the origin
# changes, but keeps the Cookie header because the domain remains the
# same.
different_port_response = self.get_response(
original_request, "https://example.com:8080/a"
)
different_port_redirect_request = self.mw.process_response(
original_request, different_port_response
)
assert isinstance(different_port_redirect_request, Request)
assert {
**safe_headers,
**cookie_header,
} == different_port_redirect_request.headers.to_unicode_dict()
# A domain change drops both the Authorization and the Cookie header.
external_response = self.get_response(
original_request, "https://example.org/a"
)
external_redirect_request = self.mw.process_response(
original_request, external_response
)
assert isinstance(external_redirect_request, Request)
assert safe_headers == external_redirect_request.headers.to_unicode_dict()
# A scheme upgrade (http → https) drops the Authorization header
# because the origin changes, but keeps the Cookie header because the
# domain remains the same.
upgrade_response = self.get_response(http_request, "https://example.com/a")
upgrade_redirect_request = self.mw.process_response(
http_request, upgrade_response
)
assert isinstance(upgrade_redirect_request, Request)
assert {
**safe_headers,
**cookie_header,
} == upgrade_redirect_request.headers.to_unicode_dict()
# A scheme downgrade (https → http) drops the Authorization header
# because the origin changes, and the Cookie header because its value
# cannot indicate whether the cookies were secure (HTTPS-only) or not.
#
# Note: If the Cookie header is set by the cookie management
# middleware, as recommended in the docs, the dropping of Cookie on
# scheme downgrade is not an issue, because the cookie management
# middleware will add again the Cookie header to the new request if
# appropriate.
downgrade_response = self.get_response(
original_request, "http://example.com/a"
)
downgrade_redirect_request = self.mw.process_response(
original_request, downgrade_response
)
assert isinstance(downgrade_redirect_request, Request)
assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict()
def test_meta_proxy_http_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_meta_proxy_http_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "/a")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "/a")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_meta_proxy_https_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_meta_proxy_https_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "/a")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "/a")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_meta_proxy_http_to_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_meta_proxy_https_to_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_http_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_http_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "/a")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "/a")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_https_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"https_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_https_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"https_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "/a")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "/a")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_proxied_http_to_proxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request2.meta["_auth_proxy"] == "https://b.example"
assert request2.meta["proxy"] == "https://b.example"
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_proxied_http_to_unproxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_unproxied_http_to_proxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert "Proxy-Authorization" not in request1.headers
assert "_auth_proxy" not in request1.meta
assert "proxy" not in request1.meta
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request2.meta["_auth_proxy"] == "https://b.example"
assert request2.meta["proxy"] == "https://b.example"
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
def test_system_proxy_unproxied_http_to_unproxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert "Proxy-Authorization" not in request1.headers
assert "_auth_proxy" not in request1.meta
assert "proxy" not in request1.meta
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
def test_system_proxy_proxied_https_to_proxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request1.meta["_auth_proxy"] == "https://b.example"
assert request1.meta["proxy"] == "https://b.example"
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request3.meta["_auth_proxy"] == "https://b.example"
assert request3.meta["proxy"] == "https://b.example"
def test_system_proxy_proxied_https_to_unproxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request1.meta["_auth_proxy"] == "https://b.example"
assert request1.meta["proxy"] == "https://b.example"
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request3.meta["_auth_proxy"] == "https://b.example"
assert request3.meta["proxy"] == "https://b.example"
def test_system_proxy_unproxied_https_to_proxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert "Proxy-Authorization" not in request1.headers
assert "_auth_proxy" not in request1.meta
assert "proxy" not in request1.meta
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
def test_system_proxy_unproxied_https_to_unproxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert "Proxy-Authorization" not in request1.headers
assert "_auth_proxy" not in request1.meta
assert "proxy" not in request1.meta
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta

4
tests/test_downloadermiddleware_redirect_metarefresh.py
View File

@ -12,7 +12,7 @@ from scrapy.http import HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.test_downloadermiddleware_redirect_base import Base
from tests.utils.bases.redirect import TestRedirectBase
from tests.utils.redirect import (
HTTP_SCHEMES,
NON_HTTP_SCHEMES,
@ -26,7 +26,7 @@ def meta_refresh_body(url, interval=5):
return html.encode("utf-8")
class TestMetaRefreshMiddleware(Base.Test):
class TestMetaRefreshMiddleware(TestRedirectBase):
mwcls = MetaRefreshMiddleware
reason = "meta refresh"

337
tests/test_engine.py
View File

@ -1,115 +1,45 @@
from __future__ import annotations
import asyncio
import re
import subprocess
import sys
from collections import defaultdict
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from unittest.mock import Mock
from urllib.parse import urlparse
import attr
import pytest
from itemadapter import ItemAdapter
from pydispatch import dispatcher
from testfixtures import LogCapture
from twisted.internet import defer
from scrapy import signals
from scrapy.core.engine import ExecutionEngine, _Slot
from scrapy.core.scheduler import BaseScheduler
from scrapy.exceptions import CloseSpider, IgnoreRequest
from scrapy.http import Headers, Request, Response
from scrapy.item import Field, Item
from scrapy.linkextractors import LinkExtractor
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.defer import (
_schedule_coro,
deferred_from_coro,
maybe_deferred_to_future,
)
from scrapy.utils.signal import disconnect_all
from scrapy.utils.defer import _schedule_coro, deferred_from_coro
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests import get_testdata
from tests.utils.bases.engine import TestEngineBase
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.engine import (
AttrsItemsSpider,
CrawlerRun,
DataClassItemsSpider,
DictItemsSpider,
MySpider,
)
if TYPE_CHECKING:
from collections.abc import AsyncIterator
from twisted.python.failure import Failure
from tests.mockserver.http import MockServer
class MyItem(Item):
name = Field()
url = Field()
price = Field()
@attr.s
class AttrsItem:
name = attr.ib(default="")
url = attr.ib(default="")
price = attr.ib(default=0)
@dataclass
class DataClassItem:
name: str = ""
url: str = ""
price: int = 0
class MySpider(Spider):
name = "scrapytest.org"
itemurl_re = re.compile(r"item\d+.html")
name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE)
price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)
item_cls: type = MyItem
def parse(self, response):
xlink = LinkExtractor()
itemre = re.compile(self.itemurl_re)
for link in xlink.extract_links(response):
if itemre.search(link.url):
yield Request(url=link.url, callback=self.parse_item)
def parse_item(self, response):
adapter = ItemAdapter(self.item_cls())
m = self.name_re.search(response.text)
if m:
adapter["name"] = m.group(1)
adapter["url"] = response.url
m = self.price_re.search(response.text)
if m:
adapter["price"] = m.group(1)
return adapter.item
class DupeFilterSpider(MySpider):
async def start(self):
for url in self.start_urls:
yield Request(url) # no dont_filter=True
class DictItemsSpider(MySpider):
item_cls = dict
class AttrsItemsSpider(MySpider):
item_cls = AttrsItem
class DataClassItemsSpider(MySpider):
item_cls = DataClassItem
class ItemZeroDivisionErrorSpider(MySpider):
custom_settings = {
"ITEM_PIPELINES": {
@ -130,253 +60,6 @@ class ChangeCloseReasonSpider(MySpider):
raise CloseSpider(reason="custom_reason")
class CrawlerRun:
"""A class to run the crawler and keep track of events occurred"""
def __init__(self, spider_class: type[Spider]):
self.respplug: list[tuple[Response, Spider]] = []
self.reqplug: list[tuple[Request, Spider]] = []
self.reqdropped: list[tuple[Request, Spider]] = []
self.reqreached: list[tuple[Request, Spider]] = []
self.itemerror: list[tuple[Any, Response, Spider, Failure]] = []
self.itemresp: list[tuple[Any, Response]] = []
self.headers: dict[Request, Headers] = {}
self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list)
self.signals_caught: dict[Any, dict[str, Any]] = {}
self.spider_class = spider_class
async def run(self, mockserver: MockServer) -> None:
self.mockserver = mockserver
start_urls = [
self.geturl("/static/"),
self.geturl("/redirect"),
self.geturl("/redirect"), # duplicate
self.geturl("/numbers"),
]
for name, signal in vars(signals).items():
if not name.startswith("_"):
dispatcher.connect(self.record_signal, signal)
self.crawler = get_crawler(self.spider_class)
self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
self.crawler.signals.connect(self.item_error, signals.item_error)
self.crawler.signals.connect(self.headers_received, signals.headers_received)
self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
self.crawler.signals.connect(
self.request_reached, signals.request_reached_downloader
)
self.crawler.signals.connect(
self.response_downloaded, signals.response_downloaded
)
self.crawler.crawl(start_urls=start_urls)
self.deferred: defer.Deferred[None] = defer.Deferred()
dispatcher.connect(self.stop, signals.engine_stopped)
await maybe_deferred_to_future(self.deferred)
async def stop(self):
for name, signal in vars(signals).items():
if not name.startswith("_"):
disconnect_all(signal)
self.deferred.callback(None)
await self.crawler.stop_async()
def geturl(self, path: str) -> str:
return self.mockserver.url(path)
def getpath(self, url: str) -> str:
u = urlparse(url)
return u.path
def item_error(
self, item: Any, response: Response, spider: Spider, failure: Failure
) -> None:
self.itemerror.append((item, response, spider, failure))
def item_scraped(self, item: Any, spider: Spider, response: Response) -> None:
self.itemresp.append((item, response))
def headers_received(
self, headers: Headers, body_length: int, request: Request, spider: Spider
) -> None:
self.headers[request] = headers
def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
self.bytes[request].append(data)
def request_scheduled(self, request: Request, spider: Spider) -> None:
self.reqplug.append((request, spider))
def request_reached(self, request: Request, spider: Spider) -> None:
self.reqreached.append((request, spider))
def request_dropped(self, request: Request, spider: Spider) -> None:
self.reqdropped.append((request, spider))
def response_downloaded(self, response: Response, spider: Spider) -> None:
self.respplug.append((response, spider))
def record_signal(self, *args: Any, **kwargs: Any) -> None:
"""Record a signal and its parameters"""
signalargs = kwargs.copy()
sig = signalargs.pop("signal")
signalargs.pop("sender", None)
self.signals_caught[sig] = signalargs
class TestEngineBase:
@staticmethod
def _assert_visited_urls(run: CrawlerRun) -> None:
must_be_visited = [
"/static/",
"/redirect",
"/redirected",
"/static/item1.html",
"/static/item2.html",
"/static/item999.html",
]
urls_visited = {rp[0].url for rp in run.respplug}
urls_expected = {run.geturl(p) for p in must_be_visited}
assert urls_expected <= urls_visited, (
f"URLs not visited: {list(urls_expected - urls_visited)}"
)
@staticmethod
def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None:
assert len(run.reqplug) == count
paths_expected = [
"/static/item999.html",
"/static/item2.html",
"/static/item1.html",
]
urls_requested = {rq[0].url for rq in run.reqplug}
urls_expected = {run.geturl(p) for p in paths_expected}
assert urls_expected <= urls_requested
scheduled_requests_count = len(run.reqplug)
dropped_requests_count = len(run.reqdropped)
responses_count = len(run.respplug)
assert scheduled_requests_count == dropped_requests_count + responses_count
assert len(run.reqreached) == responses_count
@staticmethod
def _assert_dropped_requests(run: CrawlerRun) -> None:
assert len(run.reqdropped) == 1
@staticmethod
def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None:
# response tests
assert len(run.respplug) == count
assert len(run.reqreached) == count
for response, _ in run.respplug:
if run.getpath(response.url) == "/static/item999.html":
assert response.status == 404
if run.getpath(response.url) == "/redirect":
assert response.status == 302
@staticmethod
def _assert_items_error(run: CrawlerRun) -> None:
assert len(run.itemerror) == 2
for item, response, spider, failure in run.itemerror:
assert failure.value.__class__ is ZeroDivisionError
assert spider == run.crawler.spider
assert item["url"] == response.url
if "item1.html" in item["url"]:
assert item["name"] == "Item 1 name"
assert item["price"] == "100"
if "item2.html" in item["url"]:
assert item["name"] == "Item 2 name"
assert item["price"] == "200"
@staticmethod
def _assert_scraped_items(run: CrawlerRun) -> None:
assert len(run.itemresp) == 2
for item_, response in run.itemresp:
item = ItemAdapter(item_)
assert item["url"] == response.url
if "item1.html" in item["url"]:
assert item["name"] == "Item 1 name"
assert item["price"] == "100"
if "item2.html" in item["url"]:
assert item["name"] == "Item 2 name"
assert item["price"] == "200"
@staticmethod
def _assert_headers_received(run: CrawlerRun) -> None:
for headers in run.headers.values():
assert b"Server" in headers
assert headers[b"Server"]
assert b"TwistedWeb" in headers[b"Server"]
assert b"Date" in headers
assert b"Content-Type" in headers
@staticmethod
def _assert_bytes_received(run: CrawlerRun) -> None:
assert len(run.bytes) == 9
for request, data in run.bytes.items():
joined_data = b"".join(data)
if run.getpath(request.url) == "/static/":
assert joined_data == get_testdata("test_site", "index.html")
elif run.getpath(request.url) == "/static/item1.html":
assert joined_data == get_testdata("test_site", "item1.html")
elif run.getpath(request.url) == "/static/item2.html":
assert joined_data == get_testdata("test_site", "item2.html")
elif run.getpath(request.url) == "/redirected":
assert joined_data == b"Redirected here"
elif run.getpath(request.url) == "/redirect":
assert (
joined_data == b"\n<html>\n"
b" <head>\n"
b' <meta http-equiv="refresh" content="0;URL=/redirected">\n'
b" </head>\n"
b' <body bgcolor="#FFFFFF" text="#000000">\n'
b' <a href="/redirected">click here</a>\n'
b" </body>\n"
b"</html>\n"
)
elif run.getpath(request.url) == "/static/item999.html":
assert (
joined_data == b"\n<html>\n"
b" <head><title>404 - No Such Resource</title></head>\n"
b" <body>\n"
b" <h1>No Such Resource</h1>\n"
b" <p>File not found.</p>\n"
b" </body>\n"
b"</html>\n"
)
elif run.getpath(request.url) == "/numbers":
# signal was fired multiple times
assert len(data) > 1
# bytes were received in order
numbers = [str(x).encode("utf8") for x in range(2**18)]
assert joined_data == b"".join(numbers)
@staticmethod
def _assert_signals_caught(run: CrawlerRun) -> None:
assert signals.engine_started in run.signals_caught
assert signals.engine_stopped in run.signals_caught
assert signals.spider_opened in run.signals_caught
assert signals.spider_idle in run.signals_caught
assert signals.spider_closed in run.signals_caught
assert signals.headers_received in run.signals_caught
assert {"spider": run.crawler.spider} == run.signals_caught[
signals.spider_opened
]
assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle]
assert {
"spider": run.crawler.spider,
"reason": "finished",
} == run.signals_caught[signals.spider_closed]
class TestEngine(TestEngineBase):
@coroutine_test
async def test_crawler(self, mockserver: MockServer) -> None:

6
tests/test_engine_stop_download_bytes.py
View File

@ -3,15 +3,15 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.exceptions import StopDownload
from tests.test_engine import (
from tests.utils.bases.engine import TestEngineBase
from tests.utils.decorators import coroutine_test
from tests.utils.engine import (
AttrsItemsSpider,
CrawlerRun,
DataClassItemsSpider,
DictItemsSpider,
MySpider,
TestEngineBase,
)
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
import pytest

6
tests/test_engine_stop_download_headers.py
View File

@ -3,15 +3,15 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.exceptions import StopDownload
from tests.test_engine import (
from tests.utils.bases.engine import TestEngineBase
from tests.utils.decorators import coroutine_test
from tests.utils.engine import (
AttrsItemsSpider,
CrawlerRun,
DataClassItemsSpider,
DictItemsSpider,
MySpider,
TestEngineBase,
)
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
import pytest

141
tests/test_feedexport.py
View File

@ -4,13 +4,9 @@ import csv
import json
import marshal
import pickle
import random
import shutil
import tempfile
from abc import ABC, abstractmethod
from logging import getLogger
from pathlib import Path
from string import ascii_letters, digits
from typing import IO, TYPE_CHECKING, Any
from unittest import mock
@ -32,8 +28,8 @@ from scrapy.extensions.feedexport import (
)
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import ItemSpider
from tests.utils.bases.feedexport import TestFeedExportBase
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape
@ -99,141 +95,6 @@ class LogOnStoreFileStorage:
file.close()
class TestFeedExportBase(ABC):
mockserver: MockServer
def _random_temp_filename(self, inter_dir="") -> Path:
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = "".join(chars)
return Path(self.temp_dir, inter_dir, filename)
@classmethod
def setup_class(cls):
cls.mockserver = MockServer()
cls.mockserver.__enter__()
@classmethod
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
def setup_method(self):
self.temp_dir = tempfile.mkdtemp()
def teardown_method(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
async def exported_data(
self, items: Iterable[Any], settings: dict[str, Any]
) -> dict[str, Any]:
"""
Return exported data which a spider yielding ``items`` would return.
"""
class TestSpider(scrapy.Spider):
name = "testspider"
def parse(self, response):
yield from items
return await self.run_and_export(TestSpider, settings)
async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]:
"""
Return exported data which a spider yielding no ``items`` would return.
"""
class TestSpider(scrapy.Spider):
name = "testspider"
def parse(self, response):
pass
return await self.run_and_export(TestSpider, settings)
async def assertExported(
self,
items: Iterable[Any],
header: Iterable[str],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
await self.assertExportedCsv(items, header, rows, settings)
await self.assertExportedJsonLines(items, rows, settings)
await self.assertExportedXml(items, rows, settings)
await self.assertExportedPickle(items, rows, settings)
await self.assertExportedMarshal(items, rows, settings)
await self.assertExportedMultiple(items, rows, settings)
async def assertExportedCsv( # noqa: B027
self,
items: Iterable[Any],
header: Iterable[str],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
async def assertExportedJsonLines( # noqa: B027
self,
items: Iterable[Any],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
async def assertExportedXml( # noqa: B027
self,
items: Iterable[Any],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
async def assertExportedMultiple( # noqa: B027
self,
items: Iterable[Any],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
async def assertExportedPickle( # noqa: B027
self,
items: Iterable[Any],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
async def assertExportedMarshal( # noqa: B027
self,
items: Iterable[Any],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
@abstractmethod
async def run_and_export(
self, spider_cls: type[Spider], settings: dict[str, Any]
) -> dict[str, Any]:
pass
def _load_until_eof(
self, data: bytes, load_func: Callable[[IO[bytes]], Any]
) -> list[Any]:
result: list[Any] = []
with tempfile.TemporaryFile() as temp:
temp.write(data)
temp.seek(0)
while True:
try:
result.append(load_func(temp))
except EOFError:
break
return result
class InstrumentedFeedSlot(FeedSlot):
"""Instrumented FeedSlot subclass for keeping track of calls to
start_exporting and finish_exporting."""

2
tests/test_feedexport_batch.py
View File

@ -21,7 +21,7 @@ from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
from tests.spiders import ItemSpider
from tests.test_feedexport import TestFeedExportBase
from tests.utils.bases.feedexport import TestFeedExportBase
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.feedexport import MyItem

2
tests/test_feedexport_postprocess.py
View File

@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any
import pytest
from scrapy.utils.test import get_crawler
from tests.test_feedexport import TestFeedExportBase
from tests.utils.bases.feedexport import TestFeedExportBase
from tests.utils.decorators import coroutine_test
from tests.utils.feedexport import path_to_url, printf_escape

485
tests/test_http_request.py
View File

@ -1,493 +1,18 @@
import warnings
import xmlrpc.client
from typing import Any
import pytest
from scrapy.http import Headers, Request, XmlRpcRequest
from scrapy.http.request import NO_CALLBACK
from scrapy import Request
from scrapy.http import XmlRpcRequest
from scrapy.utils.python import to_bytes
from tests.utils.bases.http_request import TestRequestBase
class TestRequest:
class TestRequest(TestRequestBase):
request_class = Request
default_method = "GET"
default_headers: dict[bytes, list[bytes]] = {}
default_meta: dict[str, Any] = {}
def test_init(self):
# Request requires url in the __init__ method
with pytest.raises(TypeError):
self.request_class()
# url argument must be basestring
with pytest.raises(TypeError):
self.request_class(123)
# priority argument must be an integer
with pytest.raises(TypeError, match="Request priority not an integer"):
self.request_class("http://www.example.com", priority="1")
r = self.request_class("http://www.example.com")
assert isinstance(r.url, str)
assert r.url == "http://www.example.com"
assert r.method == self.default_method
assert isinstance(r.headers, Headers)
assert r.headers == self.default_headers
assert r.meta == self.default_meta
meta = {"lala": "lolo"}
headers = {b"caca": b"coco"}
r = self.request_class(
"http://www.example.com", meta=meta, headers=headers, body="a body"
)
assert r.meta is not meta
assert r.meta == meta
assert r.headers is not headers
assert r.headers[b"caca"] == b"coco"
def test_url_scheme(self):
# This test passes by not raising any (ValueError) exception
self.request_class("http://example.org")
self.request_class("https://example.org")
self.request_class("s3://example.org")
self.request_class("ftp://example.org")
self.request_class("about:config")
self.request_class("data:,Hello%2C%20World!")
def test_url_no_scheme(self):
msg = "Missing scheme in request url:"
with pytest.raises(ValueError, match=msg):
self.request_class("foo")
with pytest.raises(ValueError, match=msg):
self.request_class("/foo/")
with pytest.raises(ValueError, match=msg):
self.request_class("/foo:bar")
def test_headers(self):
# Different ways of setting headers attribute
url = "http://www.scrapy.org"
headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"}
r = self.request_class(url=url, headers=headers)
p = self.request_class(url=url, headers=r.headers)
assert r.headers == p.headers
assert r.headers is not headers
assert p.headers is not r.headers
# headers must not be unicode
h = Headers({"key1": "val1", "key2": "val2"})
h["newkey"] = "newval"
for k, v in h.items():
assert isinstance(k, bytes)
for s in v:
assert isinstance(s, bytes)
def test_eq(self):
url = "http://www.scrapy.org"
r1 = self.request_class(url=url)
r2 = self.request_class(url=url)
assert r1 != r2
set_ = set()
set_.add(r1)
set_.add(r2)
assert len(set_) == 2
def test_url(self):
r = self.request_class(url="http://www.scrapy.org/path")
assert r.url == "http://www.scrapy.org/path"
def test_url_quoting(self):
r = self.request_class(url="http://www.scrapy.org/blank%20space")
assert r.url == "http://www.scrapy.org/blank%20space"
r = self.request_class(url="http://www.scrapy.org/blank space")
assert r.url == "http://www.scrapy.org/blank%20space"
def test_url_encoding(self):
r = self.request_class(url="http://www.scrapy.org/price/£")
assert r.url == "http://www.scrapy.org/price/%C2%A3"
def test_url_encoding_other(self):
# encoding affects only query part of URI, not path
# path part should always be UTF-8 encoded before percent-escaping
r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
assert r.url == "http://www.scrapy.org/price/%C2%A3"
r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
assert r.url == "http://www.scrapy.org/price/%C2%A3"
def test_url_encoding_query(self):
r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
# should be same as above
r2 = self.request_class(
url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8"
)
assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
def test_url_encoding_query_latin1(self):
# encoding is used for encoding query-string before percent-escaping;
# path is still UTF-8 encoded before percent-escaping
r3 = self.request_class(
url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1"
)
assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3"
def test_url_encoding_nonutf8_untouched(self):
# percent-escaping sequences that do not match valid UTF-8 sequences
# should be kept untouched (just upper-cased perhaps)
#
# See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2
#
# "Conversions from URIs to IRIs MUST NOT use any character encoding
# other than UTF-8 in steps 3 and 4, even if it might be possible to
# guess from the context that another character encoding than UTF-8 was
# used in the URI. For example, the URI
# "http://www.example.org/r%E9sum%E9.html" might with some guessing be
# interpreted to contain two e-acute characters encoded as iso-8859-1.
# It must not be converted to an IRI containing these e-acute
# characters. Otherwise, in the future the IRI will be mapped to
# "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
# URI from "http://www.example.org/r%E9sum%E9.html".
r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
assert r1.url == "http://www.scrapy.org/price/%a3"
r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
assert r4.url == "http://www.example.org/r%E9sum%E9.html"
def test_url_verbatim(self):
r = self.request_class(
url="http://www.scrapy.org/price/£",
meta={"verbatim_url": True},
)
assert r.url == "http://www.scrapy.org/price/£"
r = self.request_class(
url="http://www.scrapy.org/blank space",
meta={"verbatim_url": True},
)
assert r.url == "http://www.scrapy.org/blank space"
def test_body(self):
r1 = self.request_class(url="http://www.example.com/")
assert r1.body == b""
r2 = self.request_class(url="http://www.example.com/", body=b"")
assert isinstance(r2.body, bytes)
assert r2.encoding == "utf-8" # default encoding
r3 = self.request_class(
url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8"
)
assert isinstance(r3.body, bytes)
assert r3.body == b"Price: \xc2\xa3100"
r4 = self.request_class(
url="http://www.example.com/", body="Price: \xa3100", encoding="latin1"
)
assert isinstance(r4.body, bytes)
assert r4.body == b"Price: \xa3100"
def test_copy(self):
"""Test Request copy"""
def somecallback():
pass
r1 = self.request_class(
"http://www.example.com",
flags=["f1", "f2"],
callback=somecallback,
errback=somecallback,
)
r1.meta["foo"] = "bar"
r1.cb_kwargs["key"] = "value"
r2 = r1.copy()
# make sure callbaclks are copied
assert r1.callback is somecallback
assert r1.errback is somecallback
assert r2.callback is r1.callback
assert r2.errback is r1.errback
# make sure flags list is shallow copied
assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
assert r1.flags == r2.flags
# make sure cb_kwargs dict is shallow copied
assert r1.cb_kwargs is not r2.cb_kwargs, (
"cb_kwargs must be a shallow copy, not identical"
)
assert r1.cb_kwargs == r2.cb_kwargs
# make sure meta dict is shallow copied
assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
assert r1.meta == r2.meta
# make sure headers attribute is shallow copied
assert r1.headers is not r2.headers, (
"headers must be a shallow copy, not identical"
)
assert r1.headers == r2.headers
assert r1.encoding == r2.encoding
assert r1.dont_filter == r2.dont_filter
# Request.body can be identical since it's an immutable object (str)
def test_copy_inherited_classes(self):
"""Test Request children copies preserve their class"""
class CustomRequest(self.request_class):
pass
r1 = CustomRequest("http://www.example.com")
r2 = r1.copy()
assert isinstance(r2, CustomRequest)
def test_replace(self):
"""Test Request.replace() method"""
r1 = self.request_class("http://www.example.com", method="GET")
hdrs = Headers(r1.headers)
hdrs[b"key"] = b"value"
r2 = r1.replace(method="POST", body="New body", headers=hdrs)
assert r1.url == r2.url
assert (r1.method, r2.method) == ("GET", "POST")
assert (r1.body, r2.body) == (b"", b"New body")
assert (r1.headers, r2.headers) == (self.default_headers, hdrs)
# Empty attributes (which may fail if not compared properly)
r3 = self.request_class(
"http://www.example.com", meta={"a": 1}, dont_filter=True
)
r4 = r3.replace(
url="http://www.example.com/2", body=b"", meta={}, dont_filter=False
)
assert r4.url == "http://www.example.com/2"
assert r4.body == b""
assert r4.meta == {}
assert r4.dont_filter is False
# the cls argument allows changing the resulting class
custom_request_cls = type("CustomRequest", (self.request_class,), {})
r5 = r1.replace(cls=custom_request_cls)
assert isinstance(r5, custom_request_cls)
assert r5.url == r1.url
def test_method_always_str(self):
r = self.request_class("http://www.example.com", method="POST")
assert isinstance(r.method, str)
def test_immutable_attributes(self):
r = self.request_class("http://example.com")
with pytest.raises(AttributeError):
r.url = "http://example2.com"
with pytest.raises(AttributeError):
r.body = "xxx"
def test_callback_and_errback(self):
def a_function():
pass
r1 = self.request_class("http://example.com")
assert r1.callback is None
assert r1.errback is None
r2 = self.request_class("http://example.com", callback=a_function)
assert r2.callback is a_function
assert r2.errback is None
r3 = self.request_class("http://example.com", errback=a_function)
assert r3.callback is None
assert r3.errback is a_function
r4 = self.request_class(
url="http://example.com",
callback=a_function,
errback=a_function,
)
assert r4.callback is a_function
assert r4.errback is a_function
r5 = self.request_class(
url="http://example.com",
callback=NO_CALLBACK,
errback=NO_CALLBACK,
)
assert r5.callback is NO_CALLBACK
assert r5.errback is NO_CALLBACK
def test_callback_and_errback_type(self):
with pytest.raises(TypeError):
self.request_class("http://example.com", callback="a_function")
with pytest.raises(TypeError):
self.request_class("http://example.com", errback="a_function")
with pytest.raises(TypeError):
self.request_class(
url="http://example.com",
callback="a_function",
errback="a_function",
)
def test_setters(self):
request = self.request_class("http://example.com")
request.flags = ["f1"]
assert request.flags == ["f1"]
request.cookies = {"sid": "1"}
assert request.cookies == {"sid": "1"}
headers = Headers({b"X-Test": b"1"})
request.headers = headers
assert request._headers is headers
request.headers = {b"A": b"b"}
assert isinstance(request.headers, Headers)
assert request._headers[b"A"] == b"b"
def test_setter_mutable_lazy_loading(self):
"""Mutable attributes are set internally to None only until they are
read, then they always return the same falsy instance of the
corresponding mutable structure.
Setting them to None causes the next read to return a different object.
"""
request = self.request_class("http://example.com")
assert request._flags is None
assert request.flags == []
assert request.flags is request.flags
assert request._flags == []
original_flags = request.flags
request.flags = None
assert request._flags is None
assert request.flags == []
assert request.flags is not original_flags
assert request._cookies is None
assert request.cookies == {}
assert request.cookies is request.cookies
assert request._cookies == {}
original_cookies = request.cookies
request.cookies = None
assert request._cookies is None
assert request.cookies == {}
assert request.cookies is not original_cookies
if self.default_headers:
assert request._headers == self.default_headers
assert request._headers is not self.default_headers
assert request.headers == self.default_headers
else:
assert request._headers is None
assert request.headers == {}
assert request.headers is request.headers
assert isinstance(request.headers, Headers)
assert isinstance(request._headers, Headers)
original_headers = request.headers
request.headers = None
assert request._headers is None
assert request.headers == {}
assert request._headers == {}
assert request.headers is not original_headers
def test_no_callback(self):
with pytest.raises(RuntimeError):
NO_CALLBACK()
def test_from_curl(self):
# Note: more curated tests regarding curl conversion are in
# `test_utils_curl.py`
curl_command = (
"curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
"_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
"_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
" 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
"=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
"r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
"L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
"afari/537.36' -H 'Content-Type: application /x-www-form-urlencode"
"d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q="
"0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0"
"' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee"
"p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith"
"%40example.org&size=small&topping=cheese&topping=onion&delivery=1"
"2%3A15&comments=' --compressed"
)
r = self.request_class.from_curl(curl_command)
assert r.method == "POST"
assert r.url == "http://httpbin.org/post"
assert (
r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40"
b"example.org&size=small&topping=cheese&topping=onion"
b"&delivery=12%3A15&comments="
)
assert r.cookies == {
"_gauges_unique_year": "1",
"_gauges_unique": "1",
"_gauges_unique_month": "1",
"_gauges_unique_hour": "1",
"_gauges_unique_day": "1",
}
assert r.headers == {
b"Origin": [b"http://httpbin.org"],
b"Accept-Encoding": [b"gzip, deflate"],
b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"],
b"Upgrade-Insecure-Requests": [b"1"],
b"User-Agent": [
b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."
b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"
b".75 Chrome/62.0.3202.75 Safari/537.36"
],
b"Content-Type": [b"application /x-www-form-urlencoded"],
b"Accept": [
b"text/html,application/xhtml+xml,application/xml;q=0."
b"9,image/webp,image/apng,*/*;q=0.8"
],
b"Cache-Control": [b"max-age=0"],
b"Referer": [b"http://httpbin.org/forms/post"],
b"Connection": [b"keep-alive"],
}
def test_from_curl_with_kwargs(self):
r = self.request_class.from_curl(
'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"}
)
assert r.method == "POST"
assert r.meta == {"key": "value"}
def test_from_curl_ignore_unknown_options(self):
# By default: it works and ignores the unknown options: --foo and -z
with warnings.catch_warnings(): # avoid warning when executing tests
warnings.filterwarnings(
"ignore", category=UserWarning, message="Unrecognized options:"
)
r = self.request_class.from_curl(
'curl -X DELETE "http://example.org" --foo -z',
)
assert r.method == "DELETE"
# If `ignore_unknown_options` is set to `False` it raises an error with
# the unknown options: --foo and -z
with pytest.raises(ValueError, match="Unrecognized options:"):
self.request_class.from_curl(
'curl -X PATCH "http://example.org" --foo -z',
ignore_unknown_options=False,
)
class TestXmlRpcRequest(TestRequest):
class TestXmlRpcRequest(TestRequestBase):
request_class = XmlRpcRequest
default_method = "POST"
default_headers = {b"Content-Type": [b"text/xml"]}

4
tests/test_http_request_form.py
View File

@ -10,7 +10,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import FormRequest, HtmlResponse
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
from tests.test_http_request import TestRequest
from tests.utils.bases.http_request import TestRequestBase
def _buildresponse(body, **kwargs):
@ -31,7 +31,7 @@ def _qs(req, encoding="utf-8", to_unicode=False):
# FormRequest.from_response() is deprecated in favor of form2request, so the
# many tests below that exercise it ignore the resulting deprecation warning.
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestFormRequest(TestRequest):
class TestFormRequest(TestRequestBase):
request_class = FormRequest
def assertQueryEqual(self, first, second, msg=None):

4
tests/test_http_request_json.py
View File

@ -8,10 +8,10 @@ import pytest
from scrapy.http import JsonRequest
from scrapy.utils.python import to_bytes
from tests.test_http_request import TestRequest
from tests.utils.bases.http_request import TestRequestBase
class TestJsonRequest(TestRequest):
class TestJsonRequest(TestRequestBase):
request_class = JsonRequest
default_method = "GET"
default_headers = {

411
tests/test_http_response.py
View File

@ -1,413 +1,8 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from w3lib.encoding import resolve_encoding
from scrapy.exceptions import NotSupported
from scrapy.http import Headers, Request, Response
from scrapy.link import Link
from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
from tests import get_testdata
if TYPE_CHECKING:
from collections.abc import Iterable
from scrapy.http import Response
from tests.utils.bases.http_response import TestResponseBase
class TestResponse:
class TestResponse(TestResponseBase):
response_class = Response
def test_init(self):
# Response requires url in the constructor
with pytest.raises(TypeError):
self.response_class()
assert isinstance(
self.response_class("http://example.com/"), self.response_class
)
with pytest.raises(TypeError):
self.response_class(b"http://example.com")
with pytest.raises(TypeError):
self.response_class(url="http://example.com", body={})
# body can be str or None
assert isinstance(
self.response_class("http://example.com/", body=b""),
self.response_class,
)
assert isinstance(
self.response_class("http://example.com/", body=b"body"),
self.response_class,
)
# test presence of all optional parameters
assert isinstance(
self.response_class(
"http://example.com/", body=b"", headers={}, status=200
),
self.response_class,
)
r = self.response_class("http://www.example.com")
assert isinstance(r.url, str)
assert r.url == "http://www.example.com"
assert r.status == 200
assert isinstance(r.headers, Headers)
assert not r.headers
headers = {"foo": "bar"}
body = b"a body"
r = self.response_class("http://www.example.com", headers=headers, body=body)
assert r.headers is not headers
assert r.headers[b"foo"] == b"bar"
r = self.response_class("http://www.example.com", status=301)
assert r.status == 301
r = self.response_class("http://www.example.com", status="301")
assert r.status == 301
with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
self.response_class("http://example.com", status="lala200")
def test_copy(self):
"""Test Response copy"""
r1 = self.response_class("http://www.example.com", body=b"Some body")
r1.flags.append("cached")
r2 = r1.copy()
assert r1.status == r2.status
assert r1.body == r2.body
# make sure flags list is shallow copied
assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
assert r1.flags == r2.flags
# make sure headers attribute is shallow copied
assert r1.headers is not r2.headers, (
"headers must be a shallow copy, not identical"
)
assert r1.headers == r2.headers
def test_copy_meta(self):
req = Request("http://www.example.com")
req.meta["foo"] = "bar"
r1 = self.response_class(
"http://www.example.com", body=b"Some body", request=req
)
assert r1.meta is req.meta
def test_copy_cb_kwargs(self):
req = Request("http://www.example.com")
req.cb_kwargs["foo"] = "bar"
r1 = self.response_class(
"http://www.example.com", body=b"Some body", request=req
)
assert r1.cb_kwargs is req.cb_kwargs
def test_unavailable_meta(self):
r1 = self.response_class("http://www.example.com", body=b"Some body")
with pytest.raises(AttributeError, match=r"Response\.meta not available"):
r1.meta
def test_unavailable_cb_kwargs(self):
r1 = self.response_class("http://www.example.com", body=b"Some body")
with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"):
r1.cb_kwargs
def test_copy_inherited_classes(self):
"""Test Response children copies preserve their class"""
class CustomResponse(self.response_class):
pass
r1 = CustomResponse("http://www.example.com")
r2 = r1.copy()
assert isinstance(r2, CustomResponse)
def test_replace(self):
"""Test Response.replace() method"""
hdrs = Headers({"key": "value"})
r1 = self.response_class("http://www.example.com")
r2 = r1.replace(status=301, body=b"New body", headers=hdrs)
assert r1.body == b""
assert r1.url == r2.url
assert (r1.status, r2.status) == (200, 301)
assert (r1.body, r2.body) == (b"", b"New body")
assert (r1.headers, r2.headers) == ({}, hdrs)
# Empty attributes (which may fail if not compared properly)
r3 = self.response_class("http://www.example.com", flags=["cached"])
r4 = r3.replace(body=b"", flags=[])
assert r4.body == b""
assert not r4.flags
def _assert_response_values(self, response, encoding, body):
if isinstance(body, str):
body_unicode = body
body_bytes = body.encode(encoding)
else:
body_unicode = body.decode(encoding)
body_bytes = body
assert isinstance(response.body, bytes)
assert isinstance(response.text, str)
self._assert_response_encoding(response, encoding)
assert response.body == body_bytes
assert response.text == body_unicode
def _assert_response_encoding(self, response, encoding):
assert response.encoding == resolve_encoding(encoding)
def test_immutable_attributes(self):
r = self.response_class("http://example.com")
with pytest.raises(AttributeError):
r.url = "http://example2.com"
with pytest.raises(AttributeError):
r.body = "xxx"
def test_setter_mutable_lazy_loading(self):
"""Mutable attributes are set internally to None only until they are
read, then they always return the same falsy instance of the
corresponding mutable structure.
Setting them to None causes the next read to return a different object.
"""
response = self.response_class("http://example.com")
response.request = Request("http://example.com")
assert response._flags is None
assert response.flags == []
assert response.flags is response.flags
assert response._flags == []
original_flags = response.flags
response.flags = None
assert response._flags is None
assert response.flags == []
assert response.flags is not original_flags
assert response._headers is None
assert response.headers == {}
assert response.headers is response.headers
assert isinstance(response.headers, Headers)
assert isinstance(response._headers, Headers)
original_headers = response.headers
response.headers = None
assert response._headers is None
assert response.headers == {}
assert response._headers == {}
assert response.headers is not original_headers
def test_setters(self):
response = self.response_class("http://example.com")
response.flags = ["f1"]
assert response.flags == ["f1"]
headers = Headers({b"X-Test": b"1"})
response.headers = headers
assert response._headers is headers
response.headers = {b"A": b"b"}
assert isinstance(response.headers, Headers)
assert response._headers[b"A"] == b"b"
def test_urljoin(self):
"""Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
joined = self.response_class("http://www.example.com").urljoin("/test")
absolute = "http://www.example.com/test"
assert joined == absolute
def test_shortcut_attributes(self):
r = self.response_class("http://example.com", body=b"hello")
if self.response_class == Response:
msg = "Response content isn't text"
with pytest.raises(AttributeError, match=msg):
r.text
with pytest.raises(NotSupported, match=msg):
r.css("body")
with pytest.raises(NotSupported, match=msg):
r.xpath("//body")
with pytest.raises(NotSupported, match=msg):
r.jmespath("body")
else:
r.text
r.css("body")
r.xpath("//body")
# Response.follow
def test_follow_url_absolute(self):
self._assert_followed_url("http://foo.example.com", "http://foo.example.com")
def test_follow_url_relative(self):
self._assert_followed_url("foo", "http://example.com/foo")
def test_follow_link(self):
self._assert_followed_url(
Link("http://example.com/foo"), "http://example.com/foo"
)
def test_follow_None_url(self):
r = self.response_class("http://example.com")
with pytest.raises(ValueError, match="url can't be None"):
r.follow(None)
def test_follow_None_encoding(self):
r = self.response_class("http://example.com")
with pytest.raises(ValueError, match="encoding can't be None"):
r.follow("foo", encoding=None)
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
def test_follow_whitespace_url(self):
self._assert_followed_url("foo ", "http://example.com/foo")
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
def test_follow_whitespace_link(self):
self._assert_followed_url(
Link("http://example.com/foo "), "http://example.com/foo"
)
def test_follow_flags(self):
res = self.response_class("http://example.com/")
fol = res.follow("http://example.com/", flags=["cached", "allowed"])
assert fol.flags == ["cached", "allowed"]
# Response.follow_all
def test_follow_all_absolute(self):
url_list = [
"http://example.org",
"http://www.example.org",
"http://example.com",
"http://www.example.com",
]
self._assert_followed_all_urls(url_list, url_list)
def test_follow_all_relative(self):
relative = ["foo", "bar", "foo/bar", "bar/foo"]
absolute = [
"http://example.com/foo",
"http://example.com/bar",
"http://example.com/foo/bar",
"http://example.com/bar/foo",
]
self._assert_followed_all_urls(relative, absolute)
def test_follow_all_links(self):
absolute = [
"http://example.com/foo",
"http://example.com/bar",
"http://example.com/foo/bar",
"http://example.com/bar/foo",
]
links = map(Link, absolute)
self._assert_followed_all_urls(links, absolute)
def test_follow_all_empty(self):
r = self.response_class("http://example.com")
assert not list(r.follow_all([]))
def test_follow_all_invalid(self):
r = self.response_class("http://example.com")
if self.response_class == Response:
with pytest.raises(TypeError):
list(r.follow_all(urls=None))
with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
else:
with pytest.raises(
ValueError, match="Please supply exactly one of the following arguments"
):
list(r.follow_all(urls=None))
with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
def test_follow_all_whitespace(self):
relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
absolute = [
"http://example.com/foo",
"http://example.com/bar",
"http://example.com/foo/bar",
"http://example.com/bar/foo",
]
self._assert_followed_all_urls(relative, absolute)
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
def test_follow_all_whitespace_links(self):
absolute = [
"http://example.com/foo ",
"http://example.com/bar ",
"http://example.com/foo/bar ",
"http://example.com/bar/foo ",
]
links = [Link(u) for u in absolute]
expected = [u.strip() for u in absolute]
self._assert_followed_all_urls(links, expected)
def test_follow_all_flags(self):
re = self.response_class("http://www.example.com/")
urls = [
"http://www.example.com/",
"http://www.example.com/2",
"http://www.example.com/foo",
]
fol = re.follow_all(urls, flags=["cached", "allowed"])
for req in fol:
assert req.flags == ["cached", "allowed"]
def _assert_followed_url(
self,
follow_obj: str | Link,
target_url: str,
response: Response | None = None,
encoding: str | None = None,
) -> None:
if response is None:
response = self._links_response()
req = response.follow(follow_obj)
assert req.url == target_url
if encoding is not None:
assert req.encoding == encoding
def _assert_followed_all_urls(
self,
follow_obj: Iterable[str | Link],
target_urls: Iterable[str],
response: Response | None = None,
) -> None:
if response is None:
response = self._links_response()
followed = response.follow_all(follow_obj)
for req, target in zip(followed, target_urls, strict=True):
assert req.url == target
def _links_response(self) -> Response:
body = get_testdata("link_extractor", "linkextractor.html")
return self.response_class("http://example.com/index", body=body)
def _links_response_no_href(self) -> Response:
body = get_testdata("link_extractor", "linkextractor_no_href.html")
return self.response_class("http://example.com/index", body=body)

4
tests/test_http_response_text.py
View File

@ -8,10 +8,10 @@ import pytest
from scrapy.http import HtmlResponse, TextResponse, XmlResponse
from scrapy.selector import Selector
from scrapy.utils.python import to_unicode
from tests.test_http_response import TestResponse
from tests.utils.bases.http_response import TestResponseBase
class TestTextResponse(TestResponse):
class TestTextResponse(TestResponseBase):
response_class = TextResponse
def test_follow_None_encoding(self):

4
tests/test_pipeline_files.py
View File

@ -39,8 +39,8 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from .test_pipeline_media import _mocked_download_func
from .utils.cloud import mock_google_cloud_storage
from .utils.media_pipelines import mocked_download_func
# required by persist_file() and stat_file(), but as some stores don't use the argument
# we can pass this singleton to keep type hints correct
@ -94,7 +94,7 @@ class TestFilesPipeline:
settings_dict = {"FILES_STORE": self.tempdir}
crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
crawler.spider = crawler._create_spider()
crawler.engine = MagicMock(download_async=_mocked_download_func)
crawler.engine = MagicMock(download_async=mocked_download_func)
self.pipeline = FilesPipeline.from_crawler(crawler)
self.pipeline.open_spider()

14
tests/test_pipeline_media.py
View File

@ -9,7 +9,6 @@ from twisted.python.failure import Failure
from scrapy import signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException
from scrapy.pipelines.media import MediaPipeline
from scrapy.utils.defer import _defer_sleep_async
@ -18,16 +17,7 @@ from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
async def _mocked_download_func(request):
assert request.callback is NO_CALLBACK
response = request.meta.get("response")
if callable(response):
response = await response()
if isinstance(response, Exception):
raise response
return response
from tests.utils.media_pipelines import mocked_download_func
class UserDefinedPipeline(MediaPipeline):
@ -54,7 +44,7 @@ class TestBaseMediaPipeline:
def setup_method(self):
crawler = get_crawler(DefaultSpider, self.settings)
crawler.spider = crawler._create_spider()
crawler.engine = MagicMock(download_async=_mocked_download_func)
crawler.engine = MagicMock(download_async=mocked_download_func)
self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe.open_spider()
self.info = self.pipe.spiderinfo

2
tests/test_pqueues.py
View File

@ -11,7 +11,7 @@ from scrapy.spiders import Spider
from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.test import get_crawler
from tests.test_scheduler import MockDownloader
from tests.utils.downloader import MockDownloader
class TestPriorityQueue:

30
tests/test_scheduler.py
View File

@ -3,7 +3,7 @@ from __future__ import annotations
import warnings
from abc import ABC, abstractmethod
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from typing import TYPE_CHECKING, Any, NamedTuple, cast
from typing import TYPE_CHECKING
from unittest.mock import Mock
import pytest
@ -15,43 +15,17 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.defer import ensure_awaitable
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.downloader import MockDownloader
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from pathlib import Path
class MockSlot(NamedTuple):
active: list[Any]
class MockDownloader:
def __init__(self) -> None:
self.slots: dict[str, MockSlot] = {}
def get_slot_key(self, request: Request) -> str:
if Downloader.DOWNLOAD_SLOT in request.meta:
return cast("str", request.meta[Downloader.DOWNLOAD_SLOT])
return urlparse_cached(request).hostname or ""
def increment(self, slot_key: str) -> None:
slot = self.slots.setdefault(slot_key, MockSlot(active=[]))
slot.active.append(1)
def decrement(self, slot_key: str) -> None:
slot = self.slots[slot_key]
slot.active.pop()
def close(self) -> None:
pass
class MockCrawler(Crawler):
def __init__(self, priority_queue_cls: str, jobdir: Path | None):
settings = {

123
tests/test_spider.py
View File

@ -1,133 +1,18 @@
from __future__ import annotations
from typing import Any
from unittest import mock
import pytest
from testfixtures import LogCapture
from scrapy import signals
from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse, XmlResponse
from scrapy.settings import Settings
from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import get_testdata
from tests.utils.decorators import inline_callbacks_test
from tests.utils.bases.spider import TestSpiderBase
class TestSpider:
class TestSpider(TestSpiderBase):
spider_class = Spider
def test_base_spider(self):
spider = self.spider_class("example.com")
assert spider.name == "example.com"
assert spider.start_urls == []
def test_spider_args(self):
"""``__init__`` method arguments are assigned to spider attributes"""
spider = self.spider_class("example.com", foo="bar")
assert spider.foo == "bar"
def test_spider_without_name(self):
"""``__init__`` raises when the name is not provided."""
msg = "must have a name"
with pytest.raises(ValueError, match=msg):
self.spider_class()
with pytest.raises(ValueError, match=msg):
self.spider_class(somearg="foo")
def test_from_crawler_crawler_and_settings_population(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
assert hasattr(spider, "crawler")
assert spider.crawler is crawler
assert hasattr(spider, "settings")
assert spider.settings is crawler.settings
def test_from_crawler_init_call(self):
with mock.patch.object(
self.spider_class, "__init__", return_value=None
) as mock_init:
self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar")
mock_init.assert_called_once_with("example.com", foo="bar")
def test_closed_signal_call(self):
class TestSpider(self.spider_class):
closed_called = False
def closed(self, reason):
self.closed_called = True
crawler = get_crawler()
spider = TestSpider.from_crawler(crawler, "example.com")
crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider)
crawler.signals.send_catch_log(
signal=signals.spider_closed, spider=spider, reason=None
)
assert spider.closed_called
def test_update_settings(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
project_settings = {"TEST1": "project", "TEST3": "project"}
self.spider_class.custom_settings = spider_settings
settings = Settings(project_settings, priority="project")
self.spider_class.update_settings(settings)
assert settings.get("TEST1") == "spider"
assert settings.get("TEST2") == "spider"
assert settings.get("TEST3") == "project"
@inline_callbacks_test
def test_settings_in_from_crawler(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
project_settings = {
"TEST1": "project",
"TEST3": "project",
**get_reactor_settings(),
}
class TestSpider(self.spider_class):
name = "test"
custom_settings = spider_settings
@classmethod
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
spider = super().from_crawler(crawler, *args, **kwargs)
spider.settings.set("TEST1", "spider_instance", priority="spider")
return spider
crawler = Crawler(TestSpider, project_settings)
assert crawler.settings.get("TEST1") == "spider"
assert crawler.settings.get("TEST2") == "spider"
assert crawler.settings.get("TEST3") == "project"
yield crawler.crawl()
assert crawler.settings.get("TEST1") == "spider_instance"
def test_logger(self):
spider = self.spider_class("example.com")
with LogCapture() as lc:
spider.logger.info("test log msg")
lc.check(("example.com", "INFO", "test log msg"))
record = lc.records[0]
assert "spider" in record.__dict__
assert record.spider is spider
def test_log(self):
spider = self.spider_class("example.com")
with (
mock.patch("scrapy.spiders.Spider.logger") as mock_logger,
pytest.warns(
ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"
),
):
spider.log("test log msg", "INFO")
mock_logger.log.assert_called_once_with("INFO", "test log msg")
class TestXMLFeedSpider(TestSpider):
class TestXMLFeedSpider(TestSpiderBase):
spider_class = XMLFeedSpider
def test_register_namespace(self):
@ -176,7 +61,7 @@ class TestXMLFeedSpider(TestSpider):
], iterator
class TestCSVFeedSpider(TestSpider):
class TestCSVFeedSpider(TestSpiderBase):
spider_class = CSVFeedSpider
def test_parse_rows(self):

4
tests/test_spider_crawl.py
View File

@ -11,10 +11,10 @@ from scrapy.http import HtmlResponse, Request, TextResponse
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule, Spider
from scrapy.utils.test import get_crawler
from tests.test_spider import TestSpider
from tests.utils.bases.spider import TestSpiderBase
class TestCrawlSpider(TestSpider):
class TestCrawlSpider(TestSpiderBase):
test_body = b"""<html><head><title>Page title</title></head>
<body>
<p><a href="item/12.html">Item 12</a></p>

4
tests/test_spider_sitemap.py
View File

@ -14,11 +14,11 @@ from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlRespon
from scrapy.spiders import SitemapSpider
from scrapy.utils.test import get_crawler
from tests import tests_datadir
from tests.test_spider import TestSpider
from tests.utils.bases.spider import TestSpiderBase
from tests.utils.decorators import coroutine_test
class TestSitemapSpider(TestSpider):
class TestSitemapSpider(TestSpiderBase):
spider_class = SitemapSpider
BODY = b"SITEMAP"

3
tests/test_spidermiddleware_process_start.py
View File

@ -5,7 +5,6 @@ import pytest
from scrapy import Spider, signals
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from tests.test_spider_start import SLEEP_SECONDS
from .utils import twisted_sleep
from .utils.decorators import coroutine_test
@ -14,6 +13,8 @@ ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
ITEM_C = {"id": "c"}
SLEEP_SECONDS = 0.1
class AsyncioSleepSpiderMiddleware:
async def process_start(self, start):

0
tests/utils/bases/__init__.py Normal file
View File

0
tests/utils/base_commands.py → tests/utils/bases/commands.py
View File

2
tests/test_downloader_handlers_http_base.py → tests/utils/bases/download_handlers_http.py
View File

@ -1,5 +1,3 @@
"""Base classes for HTTP download handler tests."""
from __future__ import annotations
import gzip

160
tests/utils/bases/engine.py Normal file
View File

@ -0,0 +1,160 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from itemadapter import ItemAdapter
from scrapy import signals
from tests import get_testdata
if TYPE_CHECKING:
from tests.utils.engine import CrawlerRun
class TestEngineBase:
@staticmethod
def _assert_visited_urls(run: CrawlerRun) -> None:
must_be_visited = [
"/static/",
"/redirect",
"/redirected",
"/static/item1.html",
"/static/item2.html",
"/static/item999.html",
]
urls_visited = {rp[0].url for rp in run.respplug}
urls_expected = {run.geturl(p) for p in must_be_visited}
assert urls_expected <= urls_visited, (
f"URLs not visited: {list(urls_expected - urls_visited)}"
)
@staticmethod
def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None:
assert len(run.reqplug) == count
paths_expected = [
"/static/item999.html",
"/static/item2.html",
"/static/item1.html",
]
urls_requested = {rq[0].url for rq in run.reqplug}
urls_expected = {run.geturl(p) for p in paths_expected}
assert urls_expected <= urls_requested
scheduled_requests_count = len(run.reqplug)
dropped_requests_count = len(run.reqdropped)
responses_count = len(run.respplug)
assert scheduled_requests_count == dropped_requests_count + responses_count
assert len(run.reqreached) == responses_count
@staticmethod
def _assert_dropped_requests(run: CrawlerRun) -> None:
assert len(run.reqdropped) == 1
@staticmethod
def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None:
# response tests
assert len(run.respplug) == count
assert len(run.reqreached) == count
for response, _ in run.respplug:
if run.getpath(response.url) == "/static/item999.html":
assert response.status == 404
if run.getpath(response.url) == "/redirect":
assert response.status == 302
@staticmethod
def _assert_items_error(run: CrawlerRun) -> None:
assert len(run.itemerror) == 2
for item, response, spider, failure in run.itemerror:
assert failure.value.__class__ is ZeroDivisionError
assert spider == run.crawler.spider
assert item["url"] == response.url
if "item1.html" in item["url"]:
assert item["name"] == "Item 1 name"
assert item["price"] == "100"
if "item2.html" in item["url"]:
assert item["name"] == "Item 2 name"
assert item["price"] == "200"
@staticmethod
def _assert_scraped_items(run: CrawlerRun) -> None:
assert len(run.itemresp) == 2
for item_, response in run.itemresp:
item = ItemAdapter(item_)
assert item["url"] == response.url
if "item1.html" in item["url"]:
assert item["name"] == "Item 1 name"
assert item["price"] == "100"
if "item2.html" in item["url"]:
assert item["name"] == "Item 2 name"
assert item["price"] == "200"
@staticmethod
def _assert_headers_received(run: CrawlerRun) -> None:
for headers in run.headers.values():
assert b"Server" in headers
assert headers[b"Server"]
assert b"TwistedWeb" in headers[b"Server"]
assert b"Date" in headers
assert b"Content-Type" in headers
@staticmethod
def _assert_bytes_received(run: CrawlerRun) -> None:
assert len(run.bytes) == 9
for request, data in run.bytes.items():
joined_data = b"".join(data)
if run.getpath(request.url) == "/static/":
assert joined_data == get_testdata("test_site", "index.html")
elif run.getpath(request.url) == "/static/item1.html":
assert joined_data == get_testdata("test_site", "item1.html")
elif run.getpath(request.url) == "/static/item2.html":
assert joined_data == get_testdata("test_site", "item2.html")
elif run.getpath(request.url) == "/redirected":
assert joined_data == b"Redirected here"
elif run.getpath(request.url) == "/redirect":
assert (
joined_data == b"\n<html>\n"
b" <head>\n"
b' <meta http-equiv="refresh" content="0;URL=/redirected">\n'
b" </head>\n"
b' <body bgcolor="#FFFFFF" text="#000000">\n'
b' <a href="/redirected">click here</a>\n'
b" </body>\n"
b"</html>\n"
)
elif run.getpath(request.url) == "/static/item999.html":
assert (
joined_data == b"\n<html>\n"
b" <head><title>404 - No Such Resource</title></head>\n"
b" <body>\n"
b" <h1>No Such Resource</h1>\n"
b" <p>File not found.</p>\n"
b" </body>\n"
b"</html>\n"
)
elif run.getpath(request.url) == "/numbers":
# signal was fired multiple times
assert len(data) > 1
# bytes were received in order
numbers = [str(x).encode("utf8") for x in range(2**18)]
assert joined_data == b"".join(numbers)
@staticmethod
def _assert_signals_caught(run: CrawlerRun) -> None:
assert signals.engine_started in run.signals_caught
assert signals.engine_stopped in run.signals_caught
assert signals.spider_opened in run.signals_caught
assert signals.spider_idle in run.signals_caught
assert signals.spider_closed in run.signals_caught
assert signals.headers_received in run.signals_caught
assert {"spider": run.crawler.spider} == run.signals_caught[
signals.spider_opened
]
assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle]
assert {
"spider": run.crawler.spider,
"reason": "finished",
} == run.signals_caught[signals.spider_closed]

151
tests/utils/bases/feedexport.py Normal file
View File

@ -0,0 +1,151 @@
from __future__ import annotations
import random
import shutil
import tempfile
from abc import ABC, abstractmethod
from pathlib import Path
from string import ascii_letters, digits
from typing import IO, TYPE_CHECKING, Any
import scrapy
from scrapy import Spider
from tests.mockserver.http import MockServer
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
class TestFeedExportBase(ABC):
mockserver: MockServer
def _random_temp_filename(self, inter_dir="") -> Path:
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = "".join(chars)
return Path(self.temp_dir, inter_dir, filename)
@classmethod
def setup_class(cls):
cls.mockserver = MockServer()
cls.mockserver.__enter__() # pylint: disable=unnecessary-dunder-call
@classmethod
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
def setup_method(self):
self.temp_dir = tempfile.mkdtemp()
def teardown_method(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
async def exported_data(
self, items: Iterable[Any], settings: dict[str, Any]
) -> dict[str, Any]:
"""
Return exported data which a spider yielding ``items`` would return.
"""
class TestSpider(scrapy.Spider):
name = "testspider"
def parse(self, response):
yield from items
return await self.run_and_export(TestSpider, settings)
async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]:
"""
Return exported data which a spider yielding no ``items`` would return.
"""
class TestSpider(scrapy.Spider):
name = "testspider"
def parse(self, response):
pass
return await self.run_and_export(TestSpider, settings)
async def assertExported(
self,
items: Iterable[Any],
header: Iterable[str],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
await self.assertExportedCsv(items, header, rows, settings)
await self.assertExportedJsonLines(items, rows, settings)
await self.assertExportedXml(items, rows, settings)
await self.assertExportedPickle(items, rows, settings)
await self.assertExportedMarshal(items, rows, settings)
await self.assertExportedMultiple(items, rows, settings)
async def assertExportedCsv( # noqa: B027
self,
items: Iterable[Any],
header: Iterable[str],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
async def assertExportedJsonLines( # noqa: B027
self,
items: Iterable[Any],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
async def assertExportedXml( # noqa: B027
self,
items: Iterable[Any],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
async def assertExportedMultiple( # noqa: B027
self,
items: Iterable[Any],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
async def assertExportedPickle( # noqa: B027
self,
items: Iterable[Any],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
async def assertExportedMarshal( # noqa: B027
self,
items: Iterable[Any],
rows: Iterable[dict[str, Any]],
settings: dict[str, Any] | None = None,
) -> None:
pass
@abstractmethod
async def run_and_export(
self, spider_cls: type[Spider], settings: dict[str, Any]
) -> dict[str, Any]:
pass
def _load_until_eof(
self, data: bytes, load_func: Callable[[IO[bytes]], Any]
) -> list[Any]:
result: list[Any] = []
with tempfile.TemporaryFile() as temp:
temp.write(data)
temp.seek(0)
while True:
try:
result.append(load_func(temp))
except EOFError:
break
return result

490
tests/utils/bases/http_request.py Normal file
View File

@ -0,0 +1,490 @@
import warnings
from abc import ABC, abstractmethod
from typing import Any
import pytest
from scrapy.http import Headers, Request
from scrapy.http.request import NO_CALLBACK
class TestRequestBase(ABC):
default_method = "GET"
default_headers: dict[bytes, list[bytes]] = {}
default_meta: dict[str, Any] = {}
@property
@abstractmethod
def request_class(self) -> type[Request]:
raise NotImplementedError
def test_init(self):
# Request requires url in the __init__ method
with pytest.raises(TypeError):
self.request_class()
# url argument must be basestring
with pytest.raises(TypeError):
self.request_class(123)
# priority argument must be an integer
with pytest.raises(TypeError, match="Request priority not an integer"):
self.request_class("http://www.example.com", priority="1")
r = self.request_class("http://www.example.com")
assert isinstance(r.url, str)
assert r.url == "http://www.example.com"
assert r.method == self.default_method
assert isinstance(r.headers, Headers)
assert r.headers == self.default_headers
assert r.meta == self.default_meta
meta = {"lala": "lolo"}
headers = {b"caca": b"coco"}
r = self.request_class(
"http://www.example.com", meta=meta, headers=headers, body="a body"
)
assert r.meta is not meta
assert r.meta == meta
assert r.headers is not headers
assert r.headers[b"caca"] == b"coco"
def test_url_scheme(self):
# This test passes by not raising any (ValueError) exception
self.request_class("http://example.org")
self.request_class("https://example.org")
self.request_class("s3://example.org")
self.request_class("ftp://example.org")
self.request_class("about:config")
self.request_class("data:,Hello%2C%20World!")
def test_url_no_scheme(self):
msg = "Missing scheme in request url:"
with pytest.raises(ValueError, match=msg):
self.request_class("foo")
with pytest.raises(ValueError, match=msg):
self.request_class("/foo/")
with pytest.raises(ValueError, match=msg):
self.request_class("/foo:bar")
def test_headers(self):
# Different ways of setting headers attribute
url = "http://www.scrapy.org"
headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"}
r = self.request_class(url=url, headers=headers)
p = self.request_class(url=url, headers=r.headers)
assert r.headers == p.headers
assert r.headers is not headers
assert p.headers is not r.headers
# headers must not be unicode
h = Headers({"key1": "val1", "key2": "val2"})
h["newkey"] = "newval"
for k, v in h.items():
assert isinstance(k, bytes)
for s in v:
assert isinstance(s, bytes)
def test_eq(self):
url = "http://www.scrapy.org"
r1 = self.request_class(url=url)
r2 = self.request_class(url=url)
assert r1 != r2
set_ = set()
set_.add(r1)
set_.add(r2)
assert len(set_) == 2
def test_url(self):
r = self.request_class(url="http://www.scrapy.org/path")
assert r.url == "http://www.scrapy.org/path"
def test_url_quoting(self):
r = self.request_class(url="http://www.scrapy.org/blank%20space")
assert r.url == "http://www.scrapy.org/blank%20space"
r = self.request_class(url="http://www.scrapy.org/blank space")
assert r.url == "http://www.scrapy.org/blank%20space"
def test_url_encoding(self):
r = self.request_class(url="http://www.scrapy.org/price/£")
assert r.url == "http://www.scrapy.org/price/%C2%A3"
def test_url_encoding_other(self):
# encoding affects only query part of URI, not path
# path part should always be UTF-8 encoded before percent-escaping
r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
assert r.url == "http://www.scrapy.org/price/%C2%A3"
r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
assert r.url == "http://www.scrapy.org/price/%C2%A3"
def test_url_encoding_query(self):
r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
# should be same as above
r2 = self.request_class(
url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8"
)
assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
def test_url_encoding_query_latin1(self):
# encoding is used for encoding query-string before percent-escaping;
# path is still UTF-8 encoded before percent-escaping
r3 = self.request_class(
url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1"
)
assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3"
def test_url_encoding_nonutf8_untouched(self):
# percent-escaping sequences that do not match valid UTF-8 sequences
# should be kept untouched (just upper-cased perhaps)
#
# See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2
#
# "Conversions from URIs to IRIs MUST NOT use any character encoding
# other than UTF-8 in steps 3 and 4, even if it might be possible to
# guess from the context that another character encoding than UTF-8 was
# used in the URI. For example, the URI
# "http://www.example.org/r%E9sum%E9.html" might with some guessing be
# interpreted to contain two e-acute characters encoded as iso-8859-1.
# It must not be converted to an IRI containing these e-acute
# characters. Otherwise, in the future the IRI will be mapped to
# "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
# URI from "http://www.example.org/r%E9sum%E9.html".
r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
assert r1.url == "http://www.scrapy.org/price/%a3"
r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
assert r4.url == "http://www.example.org/r%E9sum%E9.html"
def test_url_verbatim(self):
r = self.request_class(
url="http://www.scrapy.org/price/£",
meta={"verbatim_url": True},
)
assert r.url == "http://www.scrapy.org/price/£"
r = self.request_class(
url="http://www.scrapy.org/blank space",
meta={"verbatim_url": True},
)
assert r.url == "http://www.scrapy.org/blank space"
def test_body(self):
r1 = self.request_class(url="http://www.example.com/")
assert r1.body == b""
r2 = self.request_class(url="http://www.example.com/", body=b"")
assert isinstance(r2.body, bytes)
assert r2.encoding == "utf-8" # default encoding
r3 = self.request_class(
url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8"
)
assert isinstance(r3.body, bytes)
assert r3.body == b"Price: \xc2\xa3100"
r4 = self.request_class(
url="http://www.example.com/", body="Price: \xa3100", encoding="latin1"
)
assert isinstance(r4.body, bytes)
assert r4.body == b"Price: \xa3100"
def test_copy(self):
"""Test Request copy"""
def somecallback():
pass
r1 = self.request_class(
"http://www.example.com",
flags=["f1", "f2"],
callback=somecallback,
errback=somecallback,
)
r1.meta["foo"] = "bar"
r1.cb_kwargs["key"] = "value"
r2 = r1.copy()
# make sure callbaclks are copied
assert r1.callback is somecallback
assert r1.errback is somecallback
assert r2.callback is r1.callback
assert r2.errback is r1.errback
# make sure flags list is shallow copied
assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
assert r1.flags == r2.flags
# make sure cb_kwargs dict is shallow copied
assert r1.cb_kwargs is not r2.cb_kwargs, (
"cb_kwargs must be a shallow copy, not identical"
)
assert r1.cb_kwargs == r2.cb_kwargs
# make sure meta dict is shallow copied
assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
assert r1.meta == r2.meta
# make sure headers attribute is shallow copied
assert r1.headers is not r2.headers, (
"headers must be a shallow copy, not identical"
)
assert r1.headers == r2.headers
assert r1.encoding == r2.encoding
assert r1.dont_filter == r2.dont_filter
# Request.body can be identical since it's an immutable object (str)
def test_copy_inherited_classes(self):
"""Test Request children copies preserve their class"""
class CustomRequest(self.request_class):
pass
r1 = CustomRequest("http://www.example.com")
r2 = r1.copy()
assert isinstance(r2, CustomRequest)
def test_replace(self):
"""Test Request.replace() method"""
r1 = self.request_class("http://www.example.com", method="GET")
hdrs = Headers(r1.headers)
hdrs[b"key"] = b"value"
r2 = r1.replace(method="POST", body="New body", headers=hdrs)
assert r1.url == r2.url
assert (r1.method, r2.method) == ("GET", "POST")
assert (r1.body, r2.body) == (b"", b"New body")
assert (r1.headers, r2.headers) == (self.default_headers, hdrs)
# Empty attributes (which may fail if not compared properly)
r3 = self.request_class(
"http://www.example.com", meta={"a": 1}, dont_filter=True
)
r4 = r3.replace(
url="http://www.example.com/2", body=b"", meta={}, dont_filter=False
)
assert r4.url == "http://www.example.com/2"
assert r4.body == b""
assert r4.meta == {}
assert r4.dont_filter is False
# the cls argument allows changing the resulting class
custom_request_cls = type("CustomRequest", (self.request_class,), {})
r5 = r1.replace(cls=custom_request_cls)
assert isinstance(r5, custom_request_cls)
assert r5.url == r1.url
def test_method_always_str(self):
r = self.request_class("http://www.example.com", method="POST")
assert isinstance(r.method, str)
def test_immutable_attributes(self):
r = self.request_class("http://example.com")
with pytest.raises(AttributeError):
r.url = "http://example2.com"
with pytest.raises(AttributeError):
r.body = "xxx"
def test_callback_and_errback(self):
def a_function():
pass
r1 = self.request_class("http://example.com")
assert r1.callback is None
assert r1.errback is None
r2 = self.request_class("http://example.com", callback=a_function)
assert r2.callback is a_function
assert r2.errback is None
r3 = self.request_class("http://example.com", errback=a_function)
assert r3.callback is None
assert r3.errback is a_function
r4 = self.request_class(
url="http://example.com",
callback=a_function,
errback=a_function,
)
assert r4.callback is a_function
assert r4.errback is a_function
r5 = self.request_class(
url="http://example.com",
callback=NO_CALLBACK,
errback=NO_CALLBACK,
)
assert r5.callback is NO_CALLBACK
assert r5.errback is NO_CALLBACK
def test_callback_and_errback_type(self):
with pytest.raises(TypeError):
self.request_class("http://example.com", callback="a_function")
with pytest.raises(TypeError):
self.request_class("http://example.com", errback="a_function")
with pytest.raises(TypeError):
self.request_class(
url="http://example.com",
callback="a_function",
errback="a_function",
)
def test_setters(self):
request = self.request_class("http://example.com")
request.flags = ["f1"]
assert request.flags == ["f1"]
request.cookies = {"sid": "1"}
assert request.cookies == {"sid": "1"}
headers = Headers({b"X-Test": b"1"})
request.headers = headers
assert request._headers is headers
request.headers = {b"A": b"b"}
assert isinstance(request.headers, Headers)
assert request._headers[b"A"] == b"b"
def test_setter_mutable_lazy_loading(self):
"""Mutable attributes are set internally to None only until they are
read, then they always return the same falsy instance of the
corresponding mutable structure.
Setting them to None causes the next read to return a different object.
"""
request = self.request_class("http://example.com")
assert request._flags is None
assert request.flags == []
assert request.flags is request.flags
assert request._flags == []
original_flags = request.flags
request.flags = None
assert request._flags is None
assert request.flags == []
assert request.flags is not original_flags
assert request._cookies is None
assert request.cookies == {}
assert request.cookies is request.cookies
assert request._cookies == {}
original_cookies = request.cookies
request.cookies = None
assert request._cookies is None
assert request.cookies == {}
assert request.cookies is not original_cookies
if self.default_headers:
assert request._headers == self.default_headers
assert request._headers is not self.default_headers
assert request.headers == self.default_headers
else:
assert request._headers is None
assert request.headers == {}
assert request.headers is request.headers
assert isinstance(request.headers, Headers)
assert isinstance(request._headers, Headers)
original_headers = request.headers
request.headers = None
assert request._headers is None
assert request.headers == {}
assert request._headers == {}
assert request.headers is not original_headers
def test_no_callback(self):
with pytest.raises(RuntimeError):
NO_CALLBACK()
def test_from_curl(self):
# Note: more curated tests regarding curl conversion are in
# `test_utils_curl.py`
curl_command = (
"curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
"_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
"_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
" 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
"=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
"r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
"L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
"afari/537.36' -H 'Content-Type: application /x-www-form-urlencode"
"d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q="
"0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0"
"' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee"
"p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith"
"%40example.org&size=small&topping=cheese&topping=onion&delivery=1"
"2%3A15&comments=' --compressed"
)
r = self.request_class.from_curl(curl_command)
assert r.method == "POST"
assert r.url == "http://httpbin.org/post"
assert (
r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40"
b"example.org&size=small&topping=cheese&topping=onion"
b"&delivery=12%3A15&comments="
)
assert r.cookies == {
"_gauges_unique_year": "1",
"_gauges_unique": "1",
"_gauges_unique_month": "1",
"_gauges_unique_hour": "1",
"_gauges_unique_day": "1",
}
assert r.headers == {
b"Origin": [b"http://httpbin.org"],
b"Accept-Encoding": [b"gzip, deflate"],
b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"],
b"Upgrade-Insecure-Requests": [b"1"],
b"User-Agent": [
b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."
b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"
b".75 Chrome/62.0.3202.75 Safari/537.36"
],
b"Content-Type": [b"application /x-www-form-urlencoded"],
b"Accept": [
b"text/html,application/xhtml+xml,application/xml;q=0."
b"9,image/webp,image/apng,*/*;q=0.8"
],
b"Cache-Control": [b"max-age=0"],
b"Referer": [b"http://httpbin.org/forms/post"],
b"Connection": [b"keep-alive"],
}
def test_from_curl_with_kwargs(self):
r = self.request_class.from_curl(
'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"}
)
assert r.method == "POST"
assert r.meta == {"key": "value"}
def test_from_curl_ignore_unknown_options(self):
# By default: it works and ignores the unknown options: --foo and -z
with warnings.catch_warnings(): # avoid warning when executing tests
warnings.filterwarnings(
"ignore", category=UserWarning, message="Unrecognized options:"
)
r = self.request_class.from_curl(
'curl -X DELETE "http://example.org" --foo -z',
)
assert r.method == "DELETE"
# If `ignore_unknown_options` is set to `False` it raises an error with
# the unknown options: --foo and -z
with pytest.raises(ValueError, match="Unrecognized options:"):
self.request_class.from_curl(
'curl -X PATCH "http://example.org" --foo -z',
ignore_unknown_options=False,
)

417
tests/utils/bases/http_response.py Normal file
View File

@ -0,0 +1,417 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
import pytest
from w3lib.encoding import resolve_encoding
from scrapy.exceptions import NotSupported
from scrapy.http import Headers, Request, Response
from scrapy.link import Link
from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
from tests import get_testdata
if TYPE_CHECKING:
from collections.abc import Iterable
class TestResponseBase(ABC):
@property
@abstractmethod
def response_class(self) -> type[Response]:
raise NotImplementedError
def test_init(self):
# Response requires url in the constructor
with pytest.raises(TypeError):
self.response_class()
assert isinstance(
self.response_class("http://example.com/"), self.response_class
)
with pytest.raises(TypeError):
self.response_class(b"http://example.com")
with pytest.raises(TypeError):
self.response_class(url="http://example.com", body={})
# body can be str or None
assert isinstance(
self.response_class("http://example.com/", body=b""),
self.response_class,
)
assert isinstance(
self.response_class("http://example.com/", body=b"body"),
self.response_class,
)
# test presence of all optional parameters
assert isinstance(
self.response_class(
"http://example.com/", body=b"", headers={}, status=200
),
self.response_class,
)
r = self.response_class("http://www.example.com")
assert isinstance(r.url, str)
assert r.url == "http://www.example.com"
assert r.status == 200
assert isinstance(r.headers, Headers)
assert not r.headers
headers = {"foo": "bar"}
body = b"a body"
r = self.response_class("http://www.example.com", headers=headers, body=body)
assert r.headers is not headers
assert r.headers[b"foo"] == b"bar"
r = self.response_class("http://www.example.com", status=301)
assert r.status == 301
r = self.response_class("http://www.example.com", status="301")
assert r.status == 301
with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
self.response_class("http://example.com", status="lala200")
def test_copy(self):
"""Test Response copy"""
r1 = self.response_class("http://www.example.com", body=b"Some body")
r1.flags.append("cached")
r2 = r1.copy()
assert r1.status == r2.status
assert r1.body == r2.body
# make sure flags list is shallow copied
assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
assert r1.flags == r2.flags
# make sure headers attribute is shallow copied
assert r1.headers is not r2.headers, (
"headers must be a shallow copy, not identical"
)
assert r1.headers == r2.headers
def test_copy_meta(self):
req = Request("http://www.example.com")
req.meta["foo"] = "bar"
r1 = self.response_class(
"http://www.example.com", body=b"Some body", request=req
)
assert r1.meta is req.meta
def test_copy_cb_kwargs(self):
req = Request("http://www.example.com")
req.cb_kwargs["foo"] = "bar"
r1 = self.response_class(
"http://www.example.com", body=b"Some body", request=req
)
assert r1.cb_kwargs is req.cb_kwargs
def test_unavailable_meta(self):
r1 = self.response_class("http://www.example.com", body=b"Some body")
with pytest.raises(AttributeError, match=r"Response\.meta not available"):
r1.meta # pylint: disable=pointless-statement
def test_unavailable_cb_kwargs(self):
r1 = self.response_class("http://www.example.com", body=b"Some body")
with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"):
r1.cb_kwargs # pylint: disable=pointless-statement
def test_copy_inherited_classes(self):
"""Test Response children copies preserve their class"""
class CustomResponse(self.response_class):
pass
r1 = CustomResponse("http://www.example.com")
r2 = r1.copy()
assert isinstance(r2, CustomResponse)
def test_replace(self):
"""Test Response.replace() method"""
hdrs = Headers({"key": "value"})
r1 = self.response_class("http://www.example.com")
r2 = r1.replace(status=301, body=b"New body", headers=hdrs)
assert r1.body == b""
assert r1.url == r2.url
assert (r1.status, r2.status) == (200, 301)
assert (r1.body, r2.body) == (b"", b"New body")
assert (r1.headers, r2.headers) == ({}, hdrs)
# Empty attributes (which may fail if not compared properly)
r3 = self.response_class("http://www.example.com", flags=["cached"])
r4 = r3.replace(body=b"", flags=[])
assert r4.body == b""
assert not r4.flags
def _assert_response_values(self, response, encoding, body):
if isinstance(body, str):
body_unicode = body
body_bytes = body.encode(encoding)
else:
body_unicode = body.decode(encoding)
body_bytes = body
assert isinstance(response.body, bytes)
assert isinstance(response.text, str)
self._assert_response_encoding(response, encoding)
assert response.body == body_bytes
assert response.text == body_unicode
def _assert_response_encoding(self, response, encoding):
assert response.encoding == resolve_encoding(encoding)
def test_immutable_attributes(self):
r = self.response_class("http://example.com")
with pytest.raises(AttributeError):
r.url = "http://example2.com"
with pytest.raises(AttributeError):
r.body = "xxx"
def test_setter_mutable_lazy_loading(self):
"""Mutable attributes are set internally to None only until they are
read, then they always return the same falsy instance of the
corresponding mutable structure.
Setting them to None causes the next read to return a different object.
"""
response = self.response_class("http://example.com")
response.request = Request("http://example.com")
assert response._flags is None
assert response.flags == []
assert response.flags is response.flags
assert response._flags == []
original_flags = response.flags
response.flags = None
assert response._flags is None
assert response.flags == []
assert response.flags is not original_flags
assert response._headers is None
assert response.headers == {}
assert response.headers is response.headers
assert isinstance(response.headers, Headers)
assert isinstance(response._headers, Headers)
original_headers = response.headers
response.headers = None
assert response._headers is None
assert response.headers == {}
assert response._headers == {}
assert response.headers is not original_headers
def test_setters(self):
response = self.response_class("http://example.com")
response.flags = ["f1"]
assert response.flags == ["f1"]
headers = Headers({b"X-Test": b"1"})
response.headers = headers
assert response._headers is headers
response.headers = {b"A": b"b"}
assert isinstance(response.headers, Headers)
assert response._headers[b"A"] == b"b"
def test_urljoin(self):
"""Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
joined = self.response_class("http://www.example.com").urljoin("/test")
absolute = "http://www.example.com/test"
assert joined == absolute
def test_shortcut_attributes(self):
r = self.response_class("http://example.com", body=b"hello")
if self.response_class == Response:
msg = "Response content isn't text"
with pytest.raises(AttributeError, match=msg):
r.text # pylint: disable=pointless-statement
with pytest.raises(NotSupported, match=msg):
r.css("body")
with pytest.raises(NotSupported, match=msg):
r.xpath("//body")
with pytest.raises(NotSupported, match=msg):
r.jmespath("body")
else:
r.text # pylint: disable=pointless-statement
r.css("body")
r.xpath("//body")
# Response.follow
def test_follow_url_absolute(self):
self._assert_followed_url("http://foo.example.com", "http://foo.example.com")
def test_follow_url_relative(self):
self._assert_followed_url("foo", "http://example.com/foo")
def test_follow_link(self):
self._assert_followed_url(
Link("http://example.com/foo"), "http://example.com/foo"
)
def test_follow_None_url(self):
r = self.response_class("http://example.com")
with pytest.raises(ValueError, match="url can't be None"):
r.follow(None)
def test_follow_None_encoding(self):
r = self.response_class("http://example.com")
with pytest.raises(ValueError, match="encoding can't be None"):
r.follow("foo", encoding=None)
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
def test_follow_whitespace_url(self):
self._assert_followed_url("foo ", "http://example.com/foo")
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
def test_follow_whitespace_link(self):
self._assert_followed_url(
Link("http://example.com/foo "), "http://example.com/foo"
)
def test_follow_flags(self):
res = self.response_class("http://example.com/")
fol = res.follow("http://example.com/", flags=["cached", "allowed"])
assert fol.flags == ["cached", "allowed"]
# Response.follow_all
def test_follow_all_absolute(self):
url_list = [
"http://example.org",
"http://www.example.org",
"http://example.com",
"http://www.example.com",
]
self._assert_followed_all_urls(url_list, url_list)
def test_follow_all_relative(self):
relative = ["foo", "bar", "foo/bar", "bar/foo"]
absolute = [
"http://example.com/foo",
"http://example.com/bar",
"http://example.com/foo/bar",
"http://example.com/bar/foo",
]
self._assert_followed_all_urls(relative, absolute)
def test_follow_all_links(self):
absolute = [
"http://example.com/foo",
"http://example.com/bar",
"http://example.com/foo/bar",
"http://example.com/bar/foo",
]
links = map(Link, absolute)
self._assert_followed_all_urls(links, absolute)
def test_follow_all_empty(self):
r = self.response_class("http://example.com")
assert not list(r.follow_all([]))
def test_follow_all_invalid(self):
r = self.response_class("http://example.com")
if self.response_class == Response:
with pytest.raises(TypeError):
list(r.follow_all(urls=None))
with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
else:
with pytest.raises(
ValueError, match="Please supply exactly one of the following arguments"
):
list(r.follow_all(urls=None))
with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
def test_follow_all_whitespace(self):
relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
absolute = [
"http://example.com/foo",
"http://example.com/bar",
"http://example.com/foo/bar",
"http://example.com/bar/foo",
]
self._assert_followed_all_urls(relative, absolute)
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
def test_follow_all_whitespace_links(self):
absolute = [
"http://example.com/foo ",
"http://example.com/bar ",
"http://example.com/foo/bar ",
"http://example.com/bar/foo ",
]
links = [Link(u) for u in absolute]
expected = [u.strip() for u in absolute]
self._assert_followed_all_urls(links, expected)
def test_follow_all_flags(self):
re = self.response_class("http://www.example.com/")
urls = [
"http://www.example.com/",
"http://www.example.com/2",
"http://www.example.com/foo",
]
fol = re.follow_all(urls, flags=["cached", "allowed"])
for req in fol:
assert req.flags == ["cached", "allowed"]
def _assert_followed_url(
self,
follow_obj: str | Link,
target_url: str,
response: Response | None = None,
encoding: str | None = None,
) -> None:
if response is None:
response = self._links_response()
req = response.follow(follow_obj)
assert req.url == target_url
if encoding is not None:
assert req.encoding == encoding
def _assert_followed_all_urls(
self,
follow_obj: Iterable[str | Link],
target_urls: Iterable[str],
response: Response | None = None,
) -> None:
if response is None:
response = self._links_response()
followed = response.follow_all(follow_obj)
for req, target in zip(followed, target_urls, strict=True):
assert req.url == target
def _links_response(self) -> Response:
body = get_testdata("link_extractor", "linkextractor.html")
return self.response_class("http://example.com/index", body=body)
def _links_response_no_href(self) -> Response:
body = get_testdata("link_extractor", "linkextractor_no_href.html")
return self.response_class("http://example.com/index", body=body)

988
tests/utils/bases/redirect.py Normal file
View File

@ -0,0 +1,988 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
import pytest
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.http import Request, Response
from scrapy.utils.misc import set_environ
from scrapy.utils.test import get_crawler
class TestRedirectBase(ABC):
mwcls: type[Any]
mw: Any
reason: int | str
@abstractmethod
def get_response(
self, request: Request, location: str, status: int = 302
) -> Response:
raise NotImplementedError
def test_priority_adjust(self):
req = Request("http://a.example")
rsp = self.get_response(req, "http://a.example/redirected")
req2 = self.mw.process_response(req, rsp)
assert req2.priority > req.priority
def test_dont_redirect(self):
url = "http://www.example.com/301"
url2 = "http://www.example.com/redirected"
req = Request(url, meta={"dont_redirect": True})
rsp = self.get_response(req, url2)
r = self.mw.process_response(req, rsp)
assert isinstance(r, Response)
assert r is rsp
# Test that it redirects when dont_redirect is False
req = Request(url, meta={"dont_redirect": False})
rsp = self.get_response(req, url2)
r = self.mw.process_response(req, rsp)
assert isinstance(r, Request)
def test_post(self):
url = "http://www.example.com/302"
url2 = "http://www.example.com/redirected2"
req = Request(
url,
method="POST",
body="test",
headers={"Content-Type": "text/plain", "Content-length": "4"},
)
rsp = self.get_response(req, url2)
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.url == url2
assert req2.method == "GET"
assert "Content-Type" not in req2.headers, (
"Content-Type header must not be present in redirected request"
)
assert "Content-Length" not in req2.headers, (
"Content-Length header must not be present in redirected request"
)
assert not req2.body, f"Redirected body must be empty, not '{req2.body!r}'"
def test_max_redirect_times(self):
self.mw.max_redirect_times = 1
req = Request("http://a.example/302")
rsp = self.get_response(req, "/redirected")
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert "redirect_times" in req.meta
assert req.meta["redirect_times"] == 1
with pytest.raises(IgnoreRequest):
self.mw.process_response(req, rsp)
def test_ttl(self):
self.mw.max_redirect_times = 100
req = Request("http://a.example/302", meta={"redirect_ttl": 1})
rsp = self.get_response(req, "/a")
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
with pytest.raises(IgnoreRequest):
self.mw.process_response(req, rsp)
def test_redirect_urls(self):
req1 = Request("http://a.example/first")
rsp1 = self.get_response(req1, "/redirected")
req2 = self.mw.process_response(req1, rsp1)
rsp2 = self.get_response(req2, "/redirected2")
req3 = self.mw.process_response(req2, rsp2)
assert req2.url == "http://a.example/redirected"
assert req2.meta["redirect_urls"] == ["http://a.example/first"]
assert req3.url == "http://a.example/redirected2"
assert req3.meta["redirect_urls"] == [
"http://a.example/first",
"http://a.example/redirected",
]
def test_redirect_reasons(self):
req1 = Request("http://a.example/first")
rsp1 = self.get_response(req1, "/redirected1")
req2 = self.mw.process_response(req1, rsp1)
rsp2 = self.get_response(req2, "/redirected2")
req3 = self.mw.process_response(req2, rsp2)
assert req2.meta["redirect_reasons"] == [self.reason]
assert req3.meta["redirect_reasons"] == [self.reason, self.reason]
def test_cross_origin_header_dropping(self):
safe_headers = {"A": "B"}
cookie_header = {"Cookie": "a=b"}
authorization_header = {"Authorization": "Bearer 123456"}
original_request = Request(
"https://example.com",
headers={**safe_headers, **cookie_header, **authorization_header},
)
# Redirects to the same origin (same scheme, same domain, same port)
# keep all headers.
internal_response = self.get_response(original_request, "https://example.com/a")
internal_redirect_request = self.mw.process_response(
original_request, internal_response
)
assert isinstance(internal_redirect_request, Request)
assert original_request.headers == internal_redirect_request.headers
# Redirects to the same origin (same scheme, same domain, same port)
# keep all headers also when the scheme is http.
http_request = Request(
"http://example.com",
headers={**safe_headers, **cookie_header, **authorization_header},
)
http_response = self.get_response(http_request, "http://example.com/a")
http_redirect_request = self.mw.process_response(http_request, http_response)
assert isinstance(http_redirect_request, Request)
assert http_request.headers == http_redirect_request.headers
# For default ports, whether the port is explicit or implicit does not
# affect the outcome, it is still the same origin.
to_explicit_port_response = self.get_response(
original_request, "https://example.com:443/a"
)
to_explicit_port_redirect_request = self.mw.process_response(
original_request, to_explicit_port_response
)
assert isinstance(to_explicit_port_redirect_request, Request)
assert original_request.headers == to_explicit_port_redirect_request.headers
# For default ports, whether the port is explicit or implicit does not
# affect the outcome, it is still the same origin.
to_implicit_port_response = self.get_response(
original_request, "https://example.com/a"
)
to_implicit_port_redirect_request = self.mw.process_response(
original_request, to_implicit_port_response
)
assert isinstance(to_implicit_port_redirect_request, Request)
assert original_request.headers == to_implicit_port_redirect_request.headers
# A port change drops the Authorization header because the origin
# changes, but keeps the Cookie header because the domain remains the
# same.
different_port_response = self.get_response(
original_request, "https://example.com:8080/a"
)
different_port_redirect_request = self.mw.process_response(
original_request, different_port_response
)
assert isinstance(different_port_redirect_request, Request)
assert {
**safe_headers,
**cookie_header,
} == different_port_redirect_request.headers.to_unicode_dict()
# A domain change drops both the Authorization and the Cookie header.
external_response = self.get_response(original_request, "https://example.org/a")
external_redirect_request = self.mw.process_response(
original_request, external_response
)
assert isinstance(external_redirect_request, Request)
assert safe_headers == external_redirect_request.headers.to_unicode_dict()
# A scheme upgrade (http → https) drops the Authorization header
# because the origin changes, but keeps the Cookie header because the
# domain remains the same.
upgrade_response = self.get_response(http_request, "https://example.com/a")
upgrade_redirect_request = self.mw.process_response(
http_request, upgrade_response
)
assert isinstance(upgrade_redirect_request, Request)
assert {
**safe_headers,
**cookie_header,
} == upgrade_redirect_request.headers.to_unicode_dict()
# A scheme downgrade (https → http) drops the Authorization header
# because the origin changes, and the Cookie header because its value
# cannot indicate whether the cookies were secure (HTTPS-only) or not.
#
# Note: If the Cookie header is set by the cookie management
# middleware, as recommended in the docs, the dropping of Cookie on
# scheme downgrade is not an issue, because the cookie management
# middleware will add again the Cookie header to the new request if
# appropriate.
downgrade_response = self.get_response(original_request, "http://example.com/a")
downgrade_redirect_request = self.mw.process_response(
original_request, downgrade_response
)
assert isinstance(downgrade_redirect_request, Request)
assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict()
def test_meta_proxy_http_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_meta_proxy_http_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "/a")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "/a")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_meta_proxy_https_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_meta_proxy_https_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "/a")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "/a")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_meta_proxy_http_to_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_meta_proxy_https_to_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_http_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_http_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "/a")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "/a")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_https_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"https_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_https_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"https_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "/a")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "/a")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_proxied_http_to_proxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request2.meta["_auth_proxy"] == "https://b.example"
assert request2.meta["proxy"] == "https://b.example"
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_proxied_http_to_unproxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
assert request1.meta["_auth_proxy"] == "https://a.example"
assert request1.meta["proxy"] == "https://a.example"
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
assert request3.meta["_auth_proxy"] == "https://a.example"
assert request3.meta["proxy"] == "https://a.example"
def test_system_proxy_unproxied_http_to_proxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert "Proxy-Authorization" not in request1.headers
assert "_auth_proxy" not in request1.meta
assert "proxy" not in request1.meta
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request2.meta["_auth_proxy"] == "https://b.example"
assert request2.meta["proxy"] == "https://b.example"
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
def test_system_proxy_unproxied_http_to_unproxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
assert "Proxy-Authorization" not in request1.headers
assert "_auth_proxy" not in request1.meta
assert "proxy" not in request1.meta
response1 = self.get_response(request1, "https://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
response2 = self.get_response(request2, "http://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
def test_system_proxy_proxied_https_to_proxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request1.meta["_auth_proxy"] == "https://b.example"
assert request1.meta["proxy"] == "https://b.example"
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request3.meta["_auth_proxy"] == "https://b.example"
assert request3.meta["proxy"] == "https://b.example"
def test_system_proxy_proxied_https_to_unproxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request1.meta["_auth_proxy"] == "https://b.example"
assert request1.meta["proxy"] == "https://b.example"
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
assert request3.meta["_auth_proxy"] == "https://b.example"
assert request3.meta["proxy"] == "https://b.example"
def test_system_proxy_unproxied_https_to_proxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert "Proxy-Authorization" not in request1.headers
assert "_auth_proxy" not in request1.meta
assert "proxy" not in request1.meta
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
assert request2.meta["_auth_proxy"] == "https://a.example"
assert request2.meta["proxy"] == "https://a.example"
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
def test_system_proxy_unproxied_https_to_unproxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
assert "Proxy-Authorization" not in request1.headers
assert "_auth_proxy" not in request1.meta
assert "proxy" not in request1.meta
response1 = self.get_response(request1, "http://example.com")
request2 = redirect_mw.process_response(request1, response1)
assert isinstance(request2, Request)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
proxy_mw.process_request(request2)
assert "Proxy-Authorization" not in request2.headers
assert "_auth_proxy" not in request2.meta
assert "proxy" not in request2.meta
response2 = self.get_response(request2, "https://example.com")
request3 = redirect_mw.process_response(request2, response2)
assert isinstance(request3, Request)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta
proxy_mw.process_request(request3)
assert "Proxy-Authorization" not in request3.headers
assert "_auth_proxy" not in request3.meta
assert "proxy" not in request3.meta

131
tests/utils/bases/spider.py Normal file
View File

@ -0,0 +1,131 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any
from unittest import mock
import pytest
from testfixtures import LogCapture
from scrapy import signals
from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests.utils.decorators import inline_callbacks_test
if TYPE_CHECKING:
from scrapy.spiders import Spider
class TestSpiderBase(ABC):
@property
@abstractmethod
def spider_class(self) -> type[Spider]:
raise NotImplementedError
def test_base_spider(self):
spider = self.spider_class("example.com")
assert spider.name == "example.com"
assert spider.start_urls == []
def test_spider_args(self):
"""``__init__`` method arguments are assigned to spider attributes"""
spider = self.spider_class("example.com", foo="bar")
assert spider.foo == "bar"
def test_spider_without_name(self):
"""``__init__`` raises when the name is not provided."""
msg = "must have a name"
with pytest.raises(ValueError, match=msg):
self.spider_class()
with pytest.raises(ValueError, match=msg):
self.spider_class(somearg="foo")
def test_from_crawler_crawler_and_settings_population(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
assert hasattr(spider, "crawler")
assert spider.crawler is crawler
assert hasattr(spider, "settings")
assert spider.settings is crawler.settings
def test_from_crawler_init_call(self):
with mock.patch.object(
self.spider_class, "__init__", return_value=None
) as mock_init:
self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar")
mock_init.assert_called_once_with("example.com", foo="bar")
def test_closed_signal_call(self):
class TestSpider(self.spider_class):
closed_called = False
def closed(self, reason):
self.closed_called = True
crawler = get_crawler()
spider = TestSpider.from_crawler(crawler, "example.com")
crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider)
crawler.signals.send_catch_log(
signal=signals.spider_closed, spider=spider, reason=None
)
assert spider.closed_called
def test_update_settings(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
project_settings = {"TEST1": "project", "TEST3": "project"}
self.spider_class.custom_settings = spider_settings
settings = Settings(project_settings, priority="project")
self.spider_class.update_settings(settings)
assert settings.get("TEST1") == "spider"
assert settings.get("TEST2") == "spider"
assert settings.get("TEST3") == "project"
@inline_callbacks_test
def test_settings_in_from_crawler(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
project_settings = {
"TEST1": "project",
"TEST3": "project",
**get_reactor_settings(),
}
class TestSpider(self.spider_class):
name = "test"
custom_settings = spider_settings
@classmethod
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
spider = super().from_crawler(crawler, *args, **kwargs)
spider.settings.set("TEST1", "spider_instance", priority="spider")
return spider
crawler = Crawler(TestSpider, project_settings)
assert crawler.settings.get("TEST1") == "spider"
assert crawler.settings.get("TEST2") == "spider"
assert crawler.settings.get("TEST3") == "project"
yield crawler.crawl()
assert crawler.settings.get("TEST1") == "spider_instance"
def test_logger(self):
spider = self.spider_class("example.com")
with LogCapture() as lc:
spider.logger.info("test log msg")
lc.check(("example.com", "INFO", "test log msg"))
record = lc.records[0]
assert "spider" in record.__dict__
assert record.spider is spider
def test_log(self):
spider = self.spider_class("example.com")
with (
mock.patch("scrapy.spiders.Spider.logger") as mock_logger,
pytest.warns(
ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"
),
):
spider.log("test log msg", "INFO")
mock_logger.log.assert_called_once_with("INFO", "test log msg")

35
tests/utils/downloader.py Normal file
View File

@ -0,0 +1,35 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any, NamedTuple, cast
from scrapy.core.downloader import Downloader
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
from scrapy.http import Request
class MockSlot(NamedTuple):
active: list[Any]
class MockDownloader:
def __init__(self) -> None:
self.slots: dict[str, MockSlot] = {}
def get_slot_key(self, request: Request) -> str:
if Downloader.DOWNLOAD_SLOT in request.meta:
return cast("str", request.meta[Downloader.DOWNLOAD_SLOT])
return urlparse_cached(request).hostname or ""
def increment(self, slot_key: str) -> None:
slot = self.slots.setdefault(slot_key, MockSlot(active=[]))
slot.active.append(1)
def decrement(self, slot_key: str) -> None:
slot = self.slots[slot_key]
slot.active.pop()
def close(self) -> None:
pass

184
tests/utils/engine.py Normal file
View File

@ -0,0 +1,184 @@
from __future__ import annotations
import re
from collections import defaultdict
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse
import attr
from itemadapter import ItemAdapter
from pydispatch import dispatcher
from twisted.internet import defer
from scrapy import signals
from scrapy.http import Headers, Request, Response
from scrapy.item import Field, Item
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.signal import disconnect_all
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
from twisted.python.failure import Failure
from tests.mockserver.http import MockServer
class MyItem(Item):
name = Field()
url = Field()
price = Field()
@attr.s
class AttrsItem:
name = attr.ib(default="")
url = attr.ib(default="")
price = attr.ib(default=0)
@dataclass
class DataClassItem:
name: str = ""
url: str = ""
price: int = 0
class MySpider(Spider):
name = "scrapytest.org"
itemurl_re = re.compile(r"item\d+.html")
name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE)
price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)
item_cls: type = MyItem
def parse(self, response):
xlink = LinkExtractor()
itemre = re.compile(self.itemurl_re)
for link in xlink.extract_links(response):
if itemre.search(link.url):
yield Request(url=link.url, callback=self.parse_item)
def parse_item(self, response):
adapter = ItemAdapter(self.item_cls())
m = self.name_re.search(response.text)
if m:
adapter["name"] = m.group(1)
adapter["url"] = response.url
m = self.price_re.search(response.text)
if m:
adapter["price"] = m.group(1)
return adapter.item
class DictItemsSpider(MySpider):
item_cls = dict
class AttrsItemsSpider(MySpider):
item_cls = AttrsItem
class DataClassItemsSpider(MySpider):
item_cls = DataClassItem
class CrawlerRun:
"""A class to run the crawler and keep track of events occurred"""
def __init__(self, spider_class: type[Spider]):
self.respplug: list[tuple[Response, Spider]] = []
self.reqplug: list[tuple[Request, Spider]] = []
self.reqdropped: list[tuple[Request, Spider]] = []
self.reqreached: list[tuple[Request, Spider]] = []
self.itemerror: list[tuple[Any, Response, Spider, Failure]] = []
self.itemresp: list[tuple[Any, Response]] = []
self.headers: dict[Request, Headers] = {}
self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list)
self.signals_caught: dict[Any, dict[str, Any]] = {}
self.spider_class = spider_class
async def run(self, mockserver: MockServer) -> None:
self.mockserver = mockserver
start_urls = [
self.geturl("/static/"),
self.geturl("/redirect"),
self.geturl("/redirect"), # duplicate
self.geturl("/numbers"),
]
for name, signal in vars(signals).items():
if not name.startswith("_"):
dispatcher.connect(self.record_signal, signal)
self.crawler = get_crawler(self.spider_class)
self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
self.crawler.signals.connect(self.item_error, signals.item_error)
self.crawler.signals.connect(self.headers_received, signals.headers_received)
self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
self.crawler.signals.connect(
self.request_reached, signals.request_reached_downloader
)
self.crawler.signals.connect(
self.response_downloaded, signals.response_downloaded
)
self.crawler.crawl(start_urls=start_urls)
self.deferred: defer.Deferred[None] = defer.Deferred()
dispatcher.connect(self.stop, signals.engine_stopped)
await maybe_deferred_to_future(self.deferred)
async def stop(self):
for name, signal in vars(signals).items():
if not name.startswith("_"):
disconnect_all(signal)
self.deferred.callback(None)
await self.crawler.stop_async()
def geturl(self, path: str) -> str:
return self.mockserver.url(path)
def getpath(self, url: str) -> str:
u = urlparse(url)
return u.path
def item_error(
self, item: Any, response: Response, spider: Spider, failure: Failure
) -> None:
self.itemerror.append((item, response, spider, failure))
def item_scraped(self, item: Any, spider: Spider, response: Response) -> None:
self.itemresp.append((item, response))
def headers_received(
self, headers: Headers, body_length: int, request: Request, spider: Spider
) -> None:
self.headers[request] = headers
def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
self.bytes[request].append(data)
def request_scheduled(self, request: Request, spider: Spider) -> None:
self.reqplug.append((request, spider))
def request_reached(self, request: Request, spider: Spider) -> None:
self.reqreached.append((request, spider))
def request_dropped(self, request: Request, spider: Spider) -> None:
self.reqdropped.append((request, spider))
def response_downloaded(self, response: Response, spider: Spider) -> None:
self.respplug.append((response, spider))
def record_signal(self, *args: Any, **kwargs: Any) -> None:
"""Record a signal and its parameters"""
signalargs = kwargs.copy()
sig = signalargs.pop("signal")
signalargs.pop("sender", None)
self.signals_caught[sig] = signalargs

15
tests/utils/media_pipelines.py Normal file
View File

@ -0,0 +1,15 @@
from __future__ import annotations
from typing import Any
from scrapy.http.request import NO_CALLBACK, Request
async def mocked_download_func(request: Request) -> Any:
assert request.callback is NO_CALLBACK
response = request.meta.get("response")
if callable(response):
response = await response()
if isinstance(response, Exception):
raise response
return response