RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Split long test files (#7329)

This commit is contained in:
Andrey Rakhmatullin 2026-03-13 14:30:25 +05:00 committed by GitHub
parent 4d2071f7b3
commit 584d99af30
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 5984 additions and 5847 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

519
tests/test_crawler.py
View File

@ -1,21 +1,13 @@
import asyncio import asyncio
import logging import logging
import platform
import re import re
import signal
import subprocess
import sys
import warnings import warnings
from abc import ABC, abstractmethod
from collections.abc import Generator from collections.abc import Generator
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
import pytest import pytest
from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from twisted.internet.defer import Deferred from twisted.internet.defer import Deferred
from w3lib import __version__ as w3lib_version
from zope.interface.exceptions import MultipleInvalid from zope.interface.exceptions import MultipleInvalid
import scrapy import scrapy
@ -30,7 +22,6 @@ from scrapy.crawler import (
from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions.throttle import AutoThrottle
from scrapy.settings import Settings, default_settings from scrapy.settings import Settings, default_settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.log import ( from scrapy.utils.log import (
_uninstall_scrapy_root_handler, _uninstall_scrapy_root_handler,
@ -39,8 +30,6 @@ from scrapy.utils.log import (
) )
from scrapy.utils.spider import DefaultSpider from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_reactor_settings from scrapy.utils.test import get_crawler, get_reactor_settings
from tests.mockserver.http import MockServer
from tests.utils import get_script_run_env
from tests.utils.decorators import coroutine_test, inline_callbacks_test from tests.utils.decorators import coroutine_test, inline_callbacks_test
BASE_SETTINGS: dict[str, Any] = {} BASE_SETTINGS: dict[str, Any] = {}
@ -764,514 +753,6 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
pytest.skip("This test is only for CrawlerRunner") pytest.skip("This test is only for CrawlerRunner")
class ScriptRunnerMixin(ABC):
@property
@abstractmethod
def script_dir(self) -> Path:
raise NotImplementedError
@staticmethod
def get_script_dir(name: str) -> Path:
return Path(__file__).parent.resolve() / name
def get_script_args(self, script_name: str, *script_args: str) -> list[str]:
script_path = self.script_dir / script_name
return [sys.executable, str(script_path), *script_args]
def run_script(self, script_name: str, *script_args: str) -> str:
args = self.get_script_args(script_name, *script_args)
p = subprocess.Popen(
args,
env=get_script_run_env(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
_, stderr = p.communicate()
return stderr.decode("utf-8")
class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
"""Common tests between CrawlerProcess and AsyncCrawlerProcess,
with the same file names and expectations.
"""
def test_simple(self):
log = self.run_script("simple.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "is_reactorless(): False" in log
def test_multi(self):
log = self.run_script("multi.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "ReactorAlreadyInstalledError" not in log
def test_reactor_default(self):
log = self.run_script("reactor_default.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the requested one "
"(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
) in log
def test_asyncio_enabled_no_reactor(self):
log = self.run_script("asyncio_enabled_no_reactor.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "RuntimeError" not in log
def test_asyncio_enabled_reactor(self):
log = self.run_script("asyncio_enabled_reactor.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "RuntimeError" not in log
@pytest.mark.skipif(
parse_version(w3lib_version) >= parse_version("2.0.0"),
reason="w3lib 2.0.0 and later do not allow invalid domains.",
)
def test_ipv6_default_name_resolver(self):
log = self.run_script("default_name_resolver.py")
assert "Spider closed (finished)" in log
assert (
"'downloader/exception_type_count/scrapy.exceptions.CannotResolveHostError': 1,"
in log
)
assert (
"scrapy.exceptions.CannotResolveHostError: DNS lookup failed: no results for hostname lookup: ::1."
in log
)
def test_caching_hostname_resolver_ipv6(self):
log = self.run_script("caching_hostname_resolver_ipv6.py")
assert "Spider closed (finished)" in log
assert "scrapy.exceptions.CannotResolveHostError" not in log
def test_caching_hostname_resolver_finite_execution(
self, mockserver: MockServer
) -> None:
log = self.run_script("caching_hostname_resolver.py", mockserver.url("/"))
assert "Spider closed (finished)" in log
assert "ERROR: Error downloading" not in log
assert "TimeoutError" not in log
assert "scrapy.exceptions.CannotResolveHostError" not in log
def test_twisted_reactor_asyncio(self):
log = self.run_script("twisted_reactor_asyncio.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
def test_twisted_reactor_asyncio_custom_settings(self):
log = self.run_script("twisted_reactor_custom_settings.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
def test_twisted_reactor_asyncio_custom_settings_same(self):
log = self.run_script("twisted_reactor_custom_settings_same.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
@pytest.mark.requires_uvloop
def test_custom_loop_asyncio(self):
log = self.run_script("asyncio_custom_loop.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_custom_loop_asyncio_deferred_signal(self):
log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" in log
assert "async pipeline opened!" in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_same_loop(self):
log = self.run_script("asyncio_enabled_reactor_same_loop.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_different_loop(self):
log = self.run_script("asyncio_enabled_reactor_different_loop.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the one specified in the ASYNCIO_EVENT_LOOP "
"setting (uvloop.Loop)"
) in log
def test_default_loop_asyncio_deferred_signal(self):
log = self.run_script("asyncio_deferred_signal.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" not in log
assert "async pipeline opened!" in log
def test_args_change_settings(self):
log = self.run_script("args_settings.py")
assert "Spider closed (finished)" in log
assert "The value of FOO is 42" in log
def test_shutdown_graceful(self):
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
args = self.get_script_args("sleeping.py", "3")
p = PopenSpawn(args, timeout=5, env=get_script_run_env())
p.expect_exact("Spider opened")
p.expect_exact("Crawled (200)")
p.kill(sig)
p.expect_exact("shutting down gracefully")
p.expect_exact("Spider closed (shutdown)")
p.wait()
@inline_callbacks_test
def test_shutdown_forced(self):
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
args = self.get_script_args("sleeping.py", "10")
p = PopenSpawn(args, timeout=5, env=get_script_run_env())
p.expect_exact("Spider opened")
p.expect_exact("Crawled (200)")
p.kill(sig)
p.expect_exact("shutting down gracefully")
# sending the second signal too fast often causes problems
d = Deferred()
call_later(0.01, d.callback, None)
yield d
p.kill(sig)
p.expect_exact("forcing unclean shutdown")
p.wait()
class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
@property
def script_dir(self) -> Path:
return self.get_script_dir("CrawlerProcess")
def test_reactor_default_twisted_reactor_select(self):
log = self.run_script("reactor_default_twisted_reactor_select.py")
if platform.system() in ["Windows", "Darwin"]:
# The goal of this test function is to test that, when a reactor is
# installed (the default one here) and a different reactor is
# configured (select here), an error raises.
#
# In Windows the default reactor is the select reactor, so that
# error does not raise.
#
# If that ever becomes the case on more platforms (i.e. if Linux
# also starts using the select reactor by default in a future
# version of Twisted), then we will need to rethink this test.
assert "Spider closed (finished)" in log
else:
assert "Spider closed (finished)" not in log
assert (
"does not match the requested one "
"(twisted.internet.selectreactor.SelectReactor)"
) in log
def test_reactor_select(self):
log = self.run_script("reactor_select.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the requested one "
"(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
) in log
def test_reactor_select_twisted_reactor_select(self):
log = self.run_script("reactor_select_twisted_reactor_select.py")
assert "Spider closed (finished)" in log
assert "ReactorAlreadyInstalledError" not in log
def test_reactor_select_subclass_twisted_reactor_select(self):
log = self.run_script("reactor_select_subclass_twisted_reactor_select.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the requested one "
"(twisted.internet.selectreactor.SelectReactor)"
) in log
def test_twisted_reactor_select(self):
log = self.run_script("twisted_reactor_select.py")
assert "Spider closed (finished)" in log
assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log
@pytest.mark.skipif(
platform.system() == "Windows", reason="PollReactor is not supported on Windows"
)
def test_twisted_reactor_poll(self):
log = self.run_script("twisted_reactor_poll.py")
assert "Spider closed (finished)" in log
assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log
def test_twisted_reactor_asyncio_custom_settings_conflict(self):
log = self.run_script("twisted_reactor_custom_settings_conflict.py")
assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log
assert (
"(twisted.internet.selectreactor.SelectReactor) does not match the requested one"
in log
)
def test_reactorless(self):
log = self.run_script("reactorless.py")
assert (
"RuntimeError: CrawlerProcess doesn't support TWISTED_ENABLED=False" in log
)
class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
@property
def script_dir(self) -> Path:
return self.get_script_dir("AsyncCrawlerProcess")
def test_twisted_reactor_custom_settings_select(self):
log = self.run_script("twisted_reactor_custom_settings_select.py")
assert "Spider closed (finished)" not in log
assert (
"(twisted.internet.asyncioreactor.AsyncioSelectorReactor) "
"does not match the requested one "
"(twisted.internet.selectreactor.SelectReactor)"
) in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_same_loop(self):
log = self.run_script("asyncio_custom_loop_custom_settings_same.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_different_loop(self):
log = self.run_script("asyncio_custom_loop_custom_settings_different.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the one specified in the ASYNCIO_EVENT_LOOP "
"setting (uvloop.Loop)"
) in log
def test_reactorless_simple(self):
log = self.run_script("reactorless_simple.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "is_reactorless(): True" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_datauri(self):
log = self.run_script("reactorless_datauri.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "{'data': 'foo'}" in log
assert "'item_scraped_count': 1" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_import_hook(self):
log = self.run_script("reactorless_import_hook.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "ImportError: Import of twisted.internet.reactor is forbidden" in log
def test_reactorless_telnetconsole_default(self):
"""By default TWISTED_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False."""
log = self.run_script("reactorless_telnetconsole_default.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "The TelnetConsole extension requires a Twisted reactor" not in log
assert "scrapy.extensions.telnet.TelnetConsole" not in log
def test_reactorless_telnetconsole_disabled(self):
"""Explicit TELNETCONSOLE_ENABLED=False, there are no warnings."""
log = self.run_script("reactorless_telnetconsole_disabled.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "The TelnetConsole extension requires a Twisted reactor" not in log
assert "scrapy.extensions.telnet.TelnetConsole" not in log
def test_reactorless_telnetconsole_enabled(self):
"""Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning."""
log = self.run_script("reactorless_telnetconsole_enabled.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "The TelnetConsole extension requires a Twisted reactor" in log
def test_reactorless_reactor(self):
log = self.run_script("reactorless_reactor.py")
assert (
"RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed"
in log
)
class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
"""Common tests between CrawlerRunner and AsyncCrawlerRunner,
with the same file names and expectations.
"""
def test_simple(self):
log = self.run_script("simple.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "is_reactorless(): False" in log
def test_multi_parallel(self):
log = self.run_script("multi_parallel.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert re.search(
r"Spider opened.+Spider opened.+Closing spider.+Closing spider",
log,
re.DOTALL,
)
def test_multi_seq(self):
log = self.run_script("multi_seq.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert re.search(
r"Spider opened.+Closing spider.+Spider opened.+Closing spider",
log,
re.DOTALL,
)
@pytest.mark.requires_uvloop
def test_custom_loop_same(self):
log = self.run_script("custom_loop_same.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_custom_loop_different(self):
log = self.run_script("custom_loop_different.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the one specified in the ASYNCIO_EVENT_LOOP "
"setting (uvloop.Loop)"
) in log
class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
@property
def script_dir(self) -> Path:
return self.get_script_dir("CrawlerRunner")
def test_explicit_default_reactor(self):
log = self.run_script("explicit_default_reactor.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
not in log
)
def test_response_ip_address(self):
log = self.run_script("ip_address.py")
assert "INFO: Spider closed (finished)" in log
assert "INFO: Host: not.a.real.domain" in log
assert "INFO: Type: <class 'ipaddress.IPv4Address'>" in log
assert "INFO: IP address: 127.0.0.1" in log
def test_change_default_reactor(self):
log = self.run_script("change_reactor.py")
assert (
"DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "DEBUG: Using asyncio event loop" in log
def test_reactorless(self):
log = self.run_script("reactorless.py")
assert (
"RuntimeError: CrawlerRunner doesn't support TWISTED_ENABLED=False" in log
)
class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
@property
def script_dir(self) -> Path:
return self.get_script_dir("AsyncCrawlerRunner")
def test_simple_default_reactor(self):
log = self.run_script("simple_default_reactor.py")
assert "Spider closed (finished)" not in log
assert (
"RuntimeError: When TWISTED_ENABLED is True, "
"AsyncCrawlerRunner requires that the installed Twisted reactor"
) in log
def test_reactorless_simple(self):
log = self.run_script("reactorless_simple.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "is_reactorless(): True" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_datauri(self):
log = self.run_script("reactorless_datauri.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "{'data': 'foo'}" in log
assert "'item_scraped_count': 1" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_reactor(self):
log = self.run_script("reactorless_reactor.py")
assert (
"RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed"
in log
)
@pytest.mark.parametrize( @pytest.mark.parametrize(
("settings", "items"), ("settings", "items"),
[ [

531
tests/test_crawler_subprocess.py Normal file
View File

@ -0,0 +1,531 @@
from __future__ import annotations
import platform
import re
import signal
import subprocess
import sys
from abc import ABC, abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from twisted.internet.defer import Deferred
from w3lib import __version__ as w3lib_version
from scrapy.utils.asyncio import call_later
from tests.utils import get_script_run_env
from tests.utils.decorators import inline_callbacks_test
if TYPE_CHECKING:
from tests.mockserver.http import MockServer
class ScriptRunnerMixin(ABC):
@property
@abstractmethod
def script_dir(self) -> Path:
raise NotImplementedError
@staticmethod
def get_script_dir(name: str) -> Path:
return Path(__file__).parent.resolve() / name
def get_script_args(self, script_name: str, *script_args: str) -> list[str]:
script_path = self.script_dir / script_name
return [sys.executable, str(script_path), *script_args]
def run_script(self, script_name: str, *script_args: str) -> str:
args = self.get_script_args(script_name, *script_args)
p = subprocess.Popen(
args,
env=get_script_run_env(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
_, stderr = p.communicate()
return stderr.decode("utf-8")
class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
"""Common tests between CrawlerProcess and AsyncCrawlerProcess,
with the same file names and expectations.
"""
def test_simple(self):
log = self.run_script("simple.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "is_reactorless(): False" in log
def test_multi(self):
log = self.run_script("multi.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "ReactorAlreadyInstalledError" not in log
def test_reactor_default(self):
log = self.run_script("reactor_default.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the requested one "
"(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
) in log
def test_asyncio_enabled_no_reactor(self):
log = self.run_script("asyncio_enabled_no_reactor.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "RuntimeError" not in log
def test_asyncio_enabled_reactor(self):
log = self.run_script("asyncio_enabled_reactor.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "RuntimeError" not in log
@pytest.mark.skipif(
parse_version(w3lib_version) >= parse_version("2.0.0"),
reason="w3lib 2.0.0 and later do not allow invalid domains.",
)
def test_ipv6_default_name_resolver(self):
log = self.run_script("default_name_resolver.py")
assert "Spider closed (finished)" in log
assert (
"'downloader/exception_type_count/scrapy.exceptions.CannotResolveHostError': 1,"
in log
)
assert (
"scrapy.exceptions.CannotResolveHostError: DNS lookup failed: no results for hostname lookup: ::1."
in log
)
def test_caching_hostname_resolver_ipv6(self):
log = self.run_script("caching_hostname_resolver_ipv6.py")
assert "Spider closed (finished)" in log
assert "scrapy.exceptions.CannotResolveHostError" not in log
def test_caching_hostname_resolver_finite_execution(
self, mockserver: MockServer
) -> None:
log = self.run_script("caching_hostname_resolver.py", mockserver.url("/"))
assert "Spider closed (finished)" in log
assert "ERROR: Error downloading" not in log
assert "TimeoutError" not in log
assert "scrapy.exceptions.CannotResolveHostError" not in log
def test_twisted_reactor_asyncio(self):
log = self.run_script("twisted_reactor_asyncio.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
def test_twisted_reactor_asyncio_custom_settings(self):
log = self.run_script("twisted_reactor_custom_settings.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
def test_twisted_reactor_asyncio_custom_settings_same(self):
log = self.run_script("twisted_reactor_custom_settings_same.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
@pytest.mark.requires_uvloop
def test_custom_loop_asyncio(self):
log = self.run_script("asyncio_custom_loop.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_custom_loop_asyncio_deferred_signal(self):
log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" in log
assert "async pipeline opened!" in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_same_loop(self):
log = self.run_script("asyncio_enabled_reactor_same_loop.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_different_loop(self):
log = self.run_script("asyncio_enabled_reactor_different_loop.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the one specified in the ASYNCIO_EVENT_LOOP "
"setting (uvloop.Loop)"
) in log
def test_default_loop_asyncio_deferred_signal(self):
log = self.run_script("asyncio_deferred_signal.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" not in log
assert "async pipeline opened!" in log
def test_args_change_settings(self):
log = self.run_script("args_settings.py")
assert "Spider closed (finished)" in log
assert "The value of FOO is 42" in log
def test_shutdown_graceful(self):
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
args = self.get_script_args("sleeping.py", "3")
p = PopenSpawn(args, timeout=5, env=get_script_run_env())
p.expect_exact("Spider opened")
p.expect_exact("Crawled (200)")
p.kill(sig)
p.expect_exact("shutting down gracefully")
p.expect_exact("Spider closed (shutdown)")
p.wait()
@inline_callbacks_test
def test_shutdown_forced(self):
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
args = self.get_script_args("sleeping.py", "10")
p = PopenSpawn(args, timeout=5, env=get_script_run_env())
p.expect_exact("Spider opened")
p.expect_exact("Crawled (200)")
p.kill(sig)
p.expect_exact("shutting down gracefully")
# sending the second signal too fast often causes problems
d = Deferred()
call_later(0.01, d.callback, None)
yield d
p.kill(sig)
p.expect_exact("forcing unclean shutdown")
p.wait()
class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
@property
def script_dir(self) -> Path:
return self.get_script_dir("CrawlerProcess")
def test_reactor_default_twisted_reactor_select(self):
log = self.run_script("reactor_default_twisted_reactor_select.py")
if platform.system() in ["Windows", "Darwin"]:
# The goal of this test function is to test that, when a reactor is
# installed (the default one here) and a different reactor is
# configured (select here), an error raises.
#
# In Windows the default reactor is the select reactor, so that
# error does not raise.
#
# If that ever becomes the case on more platforms (i.e. if Linux
# also starts using the select reactor by default in a future
# version of Twisted), then we will need to rethink this test.
assert "Spider closed (finished)" in log
else:
assert "Spider closed (finished)" not in log
assert (
"does not match the requested one "
"(twisted.internet.selectreactor.SelectReactor)"
) in log
def test_reactor_select(self):
log = self.run_script("reactor_select.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the requested one "
"(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
) in log
def test_reactor_select_twisted_reactor_select(self):
log = self.run_script("reactor_select_twisted_reactor_select.py")
assert "Spider closed (finished)" in log
assert "ReactorAlreadyInstalledError" not in log
def test_reactor_select_subclass_twisted_reactor_select(self):
log = self.run_script("reactor_select_subclass_twisted_reactor_select.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the requested one "
"(twisted.internet.selectreactor.SelectReactor)"
) in log
def test_twisted_reactor_select(self):
log = self.run_script("twisted_reactor_select.py")
assert "Spider closed (finished)" in log
assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log
@pytest.mark.skipif(
platform.system() == "Windows", reason="PollReactor is not supported on Windows"
)
def test_twisted_reactor_poll(self):
log = self.run_script("twisted_reactor_poll.py")
assert "Spider closed (finished)" in log
assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log
def test_twisted_reactor_asyncio_custom_settings_conflict(self):
log = self.run_script("twisted_reactor_custom_settings_conflict.py")
assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log
assert (
"(twisted.internet.selectreactor.SelectReactor) does not match the requested one"
in log
)
def test_reactorless(self):
log = self.run_script("reactorless.py")
assert (
"RuntimeError: CrawlerProcess doesn't support TWISTED_ENABLED=False" in log
)
class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
@property
def script_dir(self) -> Path:
return self.get_script_dir("AsyncCrawlerProcess")
def test_twisted_reactor_custom_settings_select(self):
log = self.run_script("twisted_reactor_custom_settings_select.py")
assert "Spider closed (finished)" not in log
assert (
"(twisted.internet.asyncioreactor.AsyncioSelectorReactor) "
"does not match the requested one "
"(twisted.internet.selectreactor.SelectReactor)"
) in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_same_loop(self):
log = self.run_script("asyncio_custom_loop_custom_settings_same.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_different_loop(self):
log = self.run_script("asyncio_custom_loop_custom_settings_different.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the one specified in the ASYNCIO_EVENT_LOOP "
"setting (uvloop.Loop)"
) in log
def test_reactorless_simple(self):
log = self.run_script("reactorless_simple.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "is_reactorless(): True" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_datauri(self):
log = self.run_script("reactorless_datauri.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "{'data': 'foo'}" in log
assert "'item_scraped_count': 1" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_import_hook(self):
log = self.run_script("reactorless_import_hook.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "ImportError: Import of twisted.internet.reactor is forbidden" in log
def test_reactorless_telnetconsole_default(self):
"""By default TWISTED_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False."""
log = self.run_script("reactorless_telnetconsole_default.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "The TelnetConsole extension requires a Twisted reactor" not in log
assert "scrapy.extensions.telnet.TelnetConsole" not in log
def test_reactorless_telnetconsole_disabled(self):
"""Explicit TELNETCONSOLE_ENABLED=False, there are no warnings."""
log = self.run_script("reactorless_telnetconsole_disabled.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "The TelnetConsole extension requires a Twisted reactor" not in log
assert "scrapy.extensions.telnet.TelnetConsole" not in log
def test_reactorless_telnetconsole_enabled(self):
"""Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning."""
log = self.run_script("reactorless_telnetconsole_enabled.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "The TelnetConsole extension requires a Twisted reactor" in log
def test_reactorless_reactor(self):
log = self.run_script("reactorless_reactor.py")
assert (
"RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed"
in log
)
class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
"""Common tests between CrawlerRunner and AsyncCrawlerRunner,
with the same file names and expectations.
"""
def test_simple(self):
log = self.run_script("simple.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "is_reactorless(): False" in log
def test_multi_parallel(self):
log = self.run_script("multi_parallel.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert re.search(
r"Spider opened.+Spider opened.+Closing spider.+Closing spider",
log,
re.DOTALL,
)
def test_multi_seq(self):
log = self.run_script("multi_seq.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert re.search(
r"Spider opened.+Closing spider.+Spider opened.+Closing spider",
log,
re.DOTALL,
)
@pytest.mark.requires_uvloop
def test_custom_loop_same(self):
log = self.run_script("custom_loop_same.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_custom_loop_different(self):
log = self.run_script("custom_loop_different.py")
assert "Spider closed (finished)" not in log
assert (
"does not match the one specified in the ASYNCIO_EVENT_LOOP "
"setting (uvloop.Loop)"
) in log
class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
@property
def script_dir(self) -> Path:
return self.get_script_dir("CrawlerRunner")
def test_explicit_default_reactor(self):
log = self.run_script("explicit_default_reactor.py")
assert "Spider closed (finished)" in log
assert (
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
not in log
)
def test_response_ip_address(self):
log = self.run_script("ip_address.py")
assert "INFO: Spider closed (finished)" in log
assert "INFO: Host: not.a.real.domain" in log
assert "INFO: Type: <class 'ipaddress.IPv4Address'>" in log
assert "INFO: IP address: 127.0.0.1" in log
def test_change_default_reactor(self):
log = self.run_script("change_reactor.py")
assert (
"DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "DEBUG: Using asyncio event loop" in log
def test_reactorless(self):
log = self.run_script("reactorless.py")
assert (
"RuntimeError: CrawlerRunner doesn't support TWISTED_ENABLED=False" in log
)
class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
@property
def script_dir(self) -> Path:
return self.get_script_dir("AsyncCrawlerRunner")
def test_simple_default_reactor(self):
log = self.run_script("simple_default_reactor.py")
assert "Spider closed (finished)" not in log
assert (
"RuntimeError: When TWISTED_ENABLED is True, "
"AsyncCrawlerRunner requires that the installed Twisted reactor"
) in log
def test_reactorless_simple(self):
log = self.run_script("reactorless_simple.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "is_reactorless(): True" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_datauri(self):
log = self.run_script("reactorless_datauri.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "{'data': 'foo'}" in log
assert "'item_scraped_count': 1" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_reactor(self):
log = self.run_script("reactorless_reactor.py")
assert (
"RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed"
in log
)

1169
tests/test_downloadermiddleware_redirect.py
View File

File diff suppressed because it is too large Load Diff

1023
tests/test_downloadermiddleware_redirect_base.py Normal file
View File

File diff suppressed because it is too large Load Diff

159
tests/test_downloadermiddleware_redirect_metarefresh.py Normal file
View File

@ -0,0 +1,159 @@
from __future__ import annotations
import logging
from itertools import chain
from unittest.mock import MagicMock
import pytest
from scrapy.downloadermiddlewares.redirect import MetaRefreshMiddleware
from scrapy.http import HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.test_downloadermiddleware_redirect_base import (
HTTP_SCHEMES,
NON_HTTP_SCHEMES,
REDIRECT_SCHEME_CASES,
SCHEME_PARAMS,
Base,
)
def meta_refresh_body(url, interval=5):
html = f"""<html><head><meta http-equiv="refresh" content="{interval};url={url}"/></head></html>"""
return html.encode("utf-8")
class TestMetaRefreshMiddleware(Base.Test):
mwcls = MetaRefreshMiddleware
reason = "meta refresh"
def setup_method(self):
crawler = get_crawler(Spider)
self.mw = self.mwcls.from_crawler(crawler)
def _body(self, interval=5, url="http://example.org/newpage"):
return meta_refresh_body(url, interval)
def get_response(self, request, location):
return HtmlResponse(request.url, body=self._body(url=location))
def test_meta_refresh(self):
req = Request(url="http://example.org")
rsp = HtmlResponse(req.url, body=self._body())
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.url == "http://example.org/newpage"
def test_meta_refresh_with_high_interval(self):
# meta-refresh with high intervals don't trigger redirects
req = Request(url="http://example.org")
rsp = HtmlResponse(
url="http://example.org", body=self._body(interval=1000), encoding="utf-8"
)
rsp2 = self.mw.process_response(req, rsp)
assert rsp is rsp2
def test_meta_refresh_trough_posted_request(self):
req = Request(
url="http://example.org",
method="POST",
body="test",
headers={"Content-Type": "text/plain", "Content-length": "4"},
)
rsp = HtmlResponse(req.url, body=self._body())
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.url == "http://example.org/newpage"
assert req2.method == "GET"
assert "Content-Type" not in req2.headers, (
"Content-Type header must not be present in redirected request"
)
assert "Content-Length" not in req2.headers, (
"Content-Length header must not be present in redirected request"
)
assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"
def test_ignore_tags_default(self):
req = Request(url="http://example.org")
body = (
"""<noscript><meta http-equiv="refresh" """
"""content="0;URL='http://example.org/newpage'"></noscript>"""
)
rsp = HtmlResponse(req.url, body=body.encode())
response = self.mw.process_response(req, rsp)
assert isinstance(response, Response)
def test_ignore_tags_1_x_list(self):
"""Test that Scrapy 1.x behavior remains possible"""
settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]}
crawler = get_crawler(Spider, settings)
mw = MetaRefreshMiddleware.from_crawler(crawler)
req = Request(url="http://example.org")
body = (
"""<noscript><meta http-equiv="refresh" """
"""content="0;URL='http://example.org/newpage'"></noscript>"""
)
rsp = HtmlResponse(req.url, body=body.encode())
response = mw.process_response(req, rsp)
assert isinstance(response, Response)
@pytest.mark.parametrize(
SCHEME_PARAMS,
[
*REDIRECT_SCHEME_CASES,
# data/file/ftp/s3/foo → * does not redirect
*(
(
f"{input_scheme}://example.com/a",
f"{output_scheme}://example.com/b",
None,
)
for input_scheme in NON_HTTP_SCHEMES
for output_scheme in chain(HTTP_SCHEMES, NON_HTTP_SCHEMES)
),
# data/file/ftp/s3/foo → relative does not redirect
*(
(
f"{scheme}://example.com/a",
location,
None,
)
for scheme in NON_HTTP_SCHEMES
for location in ("//example.com/b", "/b")
),
],
)
def test_meta_refresh_schemes(url, location, target):
crawler = get_crawler(Spider)
mw = MetaRefreshMiddleware.from_crawler(crawler)
request = Request(url)
response = HtmlResponse(url, body=meta_refresh_body(location))
redirect = mw.process_response(request, response)
if target is None:
assert redirect == response
else:
assert isinstance(redirect, Request)
def test_warning_meta_refresh_middleware(caplog):
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=None)
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
with caplog.at_level(logging.WARNING):
mw._engine_started()
assert (
"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware found no "
"scrapy.spidermiddlewares.referer.RefererMiddleware"
) in caplog.text
assert (
"enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)"
in caplog.text
)
assert (
"replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware "
"with a subclass that overrides the handle_referer() method"
) in caplog.text

1631
tests/test_feedexport.py
View File

File diff suppressed because it is too large Load Diff

457
tests/test_feedexport_batch.py Normal file
View File

@ -0,0 +1,457 @@
from __future__ import annotations
import csv
import json
import marshal
import pickle
from collections import defaultdict
from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin
import lxml.etree
import pytest
from packaging.version import Version
from zope.interface.verify import verifyObject
import scrapy
from scrapy import Spider
from scrapy.exceptions import NotConfigured
from scrapy.extensions.feedexport import FeedExporter, IFeedStorage, S3FeedStorage
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
from tests.spiders import ItemSpider
from tests.test_feedexport import TestFeedExportBase
from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING:
from os import PathLike
def build_url(path: str | PathLike) -> str:
path_str = str(path)
if path_str[0] != "/":
path_str = "/" + path_str
return urljoin("file:", path_str)
class TestBatchDeliveries(TestFeedExportBase):
_file_mark = "_%(batch_time)s_#%(batch_id)02d_"
async def run_and_export(
self, spider_cls: type[Spider], settings: dict[str, Any]
) -> dict[str, list[bytes]]:
"""Run spider with specified settings; return exported data."""
FEEDS = settings.get("FEEDS") or {}
settings["FEEDS"] = {
build_url(file_path): feed for file_path, feed in FEEDS.items()
}
content: defaultdict[str, list[bytes]] = defaultdict(list)
spider_cls.start_urls = [self.mockserver.url("/")]
crawler = get_crawler(spider_cls, settings)
await crawler.crawl_async()
for path, feed in FEEDS.items():
dir_name = Path(path).parent
if not dir_name.exists():
content[feed["format"]] = []
continue
for file in sorted(dir_name.iterdir()):
content[feed["format"]].append(file.read_bytes())
return content
async def assertExportedJsonLines(self, items, rows, settings=None):
settings = settings or {}
settings.update(
{
"FEEDS": {
self._random_temp_filename() / "jl" / self._file_mark: {
"format": "jl"
},
},
}
)
batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = await self.exported_data(items, settings)
for batch in data["jl"]:
got_batch = [
json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()
]
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert got_batch == expected_batch
async def assertExportedCsv(self, items, header, rows, settings=None):
settings = settings or {}
settings.update(
{
"FEEDS": {
self._random_temp_filename() / "csv" / self._file_mark: {
"format": "csv"
},
},
}
)
batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
data = await self.exported_data(items, settings)
for batch in data["csv"]:
got_batch = csv.DictReader(to_unicode(batch).splitlines())
assert list(header) == got_batch.fieldnames
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert list(got_batch) == expected_batch
async def assertExportedXml(self, items, rows, settings=None):
settings = settings or {}
settings.update(
{
"FEEDS": {
self._random_temp_filename() / "xml" / self._file_mark: {
"format": "xml"
},
},
}
)
batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = await self.exported_data(items, settings)
for batch in data["xml"]:
root = lxml.etree.fromstring(batch)
got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")]
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert got_batch == expected_batch
async def assertExportedMultiple(self, items, rows, settings=None):
settings = settings or {}
settings.update(
{
"FEEDS": {
self._random_temp_filename() / "xml" / self._file_mark: {
"format": "xml"
},
self._random_temp_filename() / "json" / self._file_mark: {
"format": "json"
},
},
}
)
batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = await self.exported_data(items, settings)
# XML
xml_rows = rows.copy()
for batch in data["xml"]:
root = lxml.etree.fromstring(batch)
got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")]
expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:]
assert got_batch == expected_batch
# JSON
json_rows = rows.copy()
for batch in data["json"]:
got_batch = json.loads(batch.decode("utf-8"))
expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:]
assert got_batch == expected_batch
async def assertExportedPickle(self, items, rows, settings=None):
settings = settings or {}
settings.update(
{
"FEEDS": {
self._random_temp_filename() / "pickle" / self._file_mark: {
"format": "pickle"
},
},
}
)
batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = await self.exported_data(items, settings)
for batch in data["pickle"]:
got_batch = self._load_until_eof(batch, load_func=pickle.load)
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert got_batch == expected_batch
async def assertExportedMarshal(self, items, rows, settings=None):
settings = settings or {}
settings.update(
{
"FEEDS": {
self._random_temp_filename() / "marshal" / self._file_mark: {
"format": "marshal"
},
},
}
)
batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = await self.exported_data(items, settings)
for batch in data["marshal"]:
got_batch = self._load_until_eof(batch, load_func=marshal.load)
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert got_batch == expected_batch
@coroutine_test
async def test_export_items(self):
"""Test partial deliveries in all supported formats"""
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
self.MyItem({"foo": "bar3", "baz": "quux3"}),
]
rows = [
{"egg": "spam1", "foo": "bar1", "baz": ""},
{"egg": "spam2", "foo": "bar2", "baz": "quux2"},
{"foo": "bar3", "baz": "quux3", "egg": ""},
]
settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2}
header = self.MyItem.fields.keys()
await self.assertExported(items, header, rows, settings=settings)
def test_wrong_path(self):
"""If path is without %(batch_time)s and %(batch_id) an exception must be raised"""
settings = {
"FEEDS": {
self._random_temp_filename(): {"format": "xml"},
},
"FEED_EXPORT_BATCH_ITEM_COUNT": 1,
}
crawler = get_crawler(settings_dict=settings)
with pytest.raises(NotConfigured):
FeedExporter(crawler)
@coroutine_test
async def test_export_no_items_not_store_empty(self):
for fmt in ("json", "jsonlines", "xml", "csv"):
settings = {
"FEEDS": {
self._random_temp_filename() / fmt / self._file_mark: {
"format": fmt
},
},
"FEED_EXPORT_BATCH_ITEM_COUNT": 1,
"FEED_STORE_EMPTY": False,
}
data = await self.exported_no_data(settings)
data = dict(data)
assert len(data[fmt]) == 0
@coroutine_test
async def test_export_no_items_store_empty(self):
formats = (
("json", b"[]"),
("jsonlines", b""),
("xml", b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
("csv", b""),
)
for fmt, expctd in formats:
settings = {
"FEEDS": {
self._random_temp_filename() / fmt / self._file_mark: {
"format": fmt
},
},
"FEED_STORE_EMPTY": True,
"FEED_EXPORT_INDENT": None,
"FEED_EXPORT_BATCH_ITEM_COUNT": 1,
}
data = await self.exported_no_data(settings)
data = dict(data)
assert data[fmt][0] == expctd
@coroutine_test
async def test_export_multiple_configs(self):
items = [
{"foo": "FOO", "bar": "BAR"},
{"foo": "FOO1", "bar": "BAR1"},
]
formats = {
"json": [
b'[\n{"bar": "BAR"}\n]',
b'[\n{"bar": "BAR1"}\n]',
],
"xml": [
(
b'<?xml version="1.0" encoding="latin-1"?>\n'
b"<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>"
),
(
b'<?xml version="1.0" encoding="latin-1"?>\n'
b"<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>"
),
],
"csv": [
b"foo,bar\r\nFOO,BAR\r\n",
b"foo,bar\r\nFOO1,BAR1\r\n",
],
}
settings = {
"FEEDS": {
self._random_temp_filename() / "json" / self._file_mark: {
"format": "json",
"indent": 0,
"fields": ["bar"],
"encoding": "utf-8",
},
self._random_temp_filename() / "xml" / self._file_mark: {
"format": "xml",
"indent": 2,
"fields": ["foo"],
"encoding": "latin-1",
},
self._random_temp_filename() / "csv" / self._file_mark: {
"format": "csv",
"indent": None,
"fields": ["foo", "bar"],
"encoding": "utf-8",
},
},
"FEED_EXPORT_BATCH_ITEM_COUNT": 1,
}
data = await self.exported_data(items, settings)
for fmt, expected in formats.items():
for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
assert got_batch == expected_batch
@coroutine_test
async def test_batch_item_count_feeds_setting(self):
items = [{"foo": "FOO"}, {"foo": "FOO1"}]
formats = {
"json": [
b'[{"foo": "FOO"}]',
b'[{"foo": "FOO1"}]',
],
}
settings = {
"FEEDS": {
self._random_temp_filename() / "json" / self._file_mark: {
"format": "json",
"indent": None,
"encoding": "utf-8",
"batch_item_count": 1,
},
},
}
data = await self.exported_data(items, settings)
for fmt, expected in formats.items():
for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
assert got_batch == expected_batch
@coroutine_test
async def test_batch_path_differ(self):
"""
Test that the name of all batch files differ from each other.
So %(batch_id)d replaced with the current id.
"""
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
self.MyItem({"foo": "bar3", "baz": "quux3"}),
]
settings = {
"FEEDS": {
self._random_temp_filename() / "%(batch_id)d": {
"format": "json",
},
},
"FEED_EXPORT_BATCH_ITEM_COUNT": 1,
}
data = await self.exported_data(items, settings)
assert len(items) == len(data["json"])
@inline_callbacks_test
def test_stats_batch_file_success(self):
settings = {
"FEEDS": {
build_url(
str(self._random_temp_filename() / "json" / self._file_mark)
): {
"format": "json",
}
},
"FEED_EXPORT_BATCH_ITEM_COUNT": 1,
}
crawler = get_crawler(ItemSpider, settings)
yield crawler.crawl(total=2, mockserver=self.mockserver)
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12
@pytest.mark.requires_boto3
@inline_callbacks_test
def test_s3_export(self):
bucket = "mybucket"
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
self.MyItem({"foo": "bar3", "baz": "quux3"}),
]
class CustomS3FeedStorage(S3FeedStorage):
stubs = []
def open(self, *args, **kwargs):
from botocore import __version__ as botocore_version # noqa: PLC0415
from botocore.stub import ANY, Stubber # noqa: PLC0415
expected_params = {
"Body": ANY,
"Bucket": bucket,
"Key": ANY,
}
if Version(botocore_version) >= Version("1.36.0"):
expected_params["ChecksumAlgorithm"] = ANY
stub = Stubber(self.s3_client)
stub.activate()
CustomS3FeedStorage.stubs.append(stub)
stub.add_response(
"put_object",
expected_params=expected_params,
service_response={},
)
return super().open(*args, **kwargs)
key = "export.csv"
uri = f"s3://{bucket}/{key}/%(batch_id)d.json"
batch_item_count = 1
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
"AWS_SECRET_ACCESS_KEY": "secret_key",
"FEED_EXPORT_BATCH_ITEM_COUNT": batch_item_count,
"FEED_STORAGES": {
"s3": CustomS3FeedStorage,
},
"FEEDS": {
uri: {
"format": "json",
},
},
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, uri)
verifyObject(IFeedStorage, storage)
class TestSpider(scrapy.Spider):
name = "testspider"
def parse(self, response):
yield from items
TestSpider.start_urls = [self.mockserver.url("/")]
crawler = get_crawler(TestSpider, settings)
yield crawler.crawl()
assert len(CustomS3FeedStorage.stubs) == len(items)
for stub in CustomS3FeedStorage.stubs[:-1]:
stub.assert_no_pending_responses()
assert (
"feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats()
)
assert (
crawler.stats.get_value("feedexport/success_count/CustomS3FeedStorage") == 3
)

538
tests/test_feedexport_postprocess.py Normal file
View File

@ -0,0 +1,538 @@
from __future__ import annotations
import bz2
import gzip
import lzma
import marshal
import pickle
import sys
from io import BytesIO
from pathlib import Path
from typing import TYPE_CHECKING, Any
import pytest
from scrapy.utils.test import get_crawler
from tests.test_feedexport import TestFeedExportBase, path_to_url, printf_escape
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy import Spider
class TestFeedPostProcessedExports(TestFeedExportBase):
items = [{"foo": "bar"}]
expected = b"foo\r\nbar\r\n"
class MyPlugin1:
def __init__(self, file, feed_options):
self.file = file
self.feed_options = feed_options
self.char = self.feed_options.get("plugin1_char", b"")
def write(self, data):
written_count = self.file.write(data)
written_count += self.file.write(self.char)
return written_count
def close(self):
self.file.close()
def _named_tempfile(self, name) -> str:
return str(Path(self.temp_dir, name))
async def run_and_export(
self, spider_cls: type[Spider], settings: dict[str, Any]
) -> dict[str, bytes | None]:
"""Run spider with specified settings; return exported data with filename."""
FEEDS = settings.get("FEEDS") or {}
settings["FEEDS"] = {
printf_escape(path_to_url(file_path)): feed_options
for file_path, feed_options in FEEDS.items()
}
content: dict[str, bytes | None] = {}
try:
spider_cls.start_urls = [self.mockserver.url("/")]
crawler = get_crawler(spider_cls, settings)
await crawler.crawl_async()
for file_path in FEEDS:
content[str(file_path)] = (
Path(file_path).read_bytes() if Path(file_path).exists() else None
)
finally:
for file_path in FEEDS:
if not Path(file_path).exists():
continue
Path(file_path).unlink()
return content
def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""):
data_stream = BytesIO()
gzipf = gzip.GzipFile(
fileobj=data_stream,
filename=filename,
mtime=mtime,
compresslevel=compresslevel,
mode="wb",
)
gzipf.write(data)
gzipf.close()
data_stream.seek(0)
return data_stream.read()
@coroutine_test
async def test_gzip_plugin(self):
filename = self._named_tempfile("gzip_file")
settings = {
"FEEDS": {
filename: {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
},
},
}
data = await self.exported_data(self.items, settings)
try:
gzip.decompress(data[filename])
except OSError:
pytest.fail("Received invalid gzip data.")
@coroutine_test
async def test_gzip_plugin_compresslevel(self):
filename_to_compressed = {
self._named_tempfile("compresslevel_0"): self.get_gzip_compressed(
self.expected, compresslevel=0
),
self._named_tempfile("compresslevel_9"): self.get_gzip_compressed(
self.expected, compresslevel=9
),
}
settings = {
"FEEDS": {
self._named_tempfile("compresslevel_0"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_compresslevel": 0,
"gzip_mtime": 0,
"gzip_filename": "",
},
self._named_tempfile("compresslevel_9"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_compresslevel": 9,
"gzip_mtime": 0,
"gzip_filename": "",
},
},
}
data = await self.exported_data(self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = gzip.decompress(data[filename])
assert compressed == data[filename]
assert result == self.expected
@coroutine_test
async def test_gzip_plugin_mtime(self):
filename_to_compressed = {
self._named_tempfile("mtime_123"): self.get_gzip_compressed(
self.expected, mtime=123
),
self._named_tempfile("mtime_123456789"): self.get_gzip_compressed(
self.expected, mtime=123456789
),
}
settings = {
"FEEDS": {
self._named_tempfile("mtime_123"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_mtime": 123,
"gzip_filename": "",
},
self._named_tempfile("mtime_123456789"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_mtime": 123456789,
"gzip_filename": "",
},
},
}
data = await self.exported_data(self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = gzip.decompress(data[filename])
assert compressed == data[filename]
assert result == self.expected
@coroutine_test
async def test_gzip_plugin_filename(self):
filename_to_compressed = {
self._named_tempfile("filename_FILE1"): self.get_gzip_compressed(
self.expected, filename="FILE1"
),
self._named_tempfile("filename_FILE2"): self.get_gzip_compressed(
self.expected, filename="FILE2"
),
}
settings = {
"FEEDS": {
self._named_tempfile("filename_FILE1"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_mtime": 0,
"gzip_filename": "FILE1",
},
self._named_tempfile("filename_FILE2"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"],
"gzip_mtime": 0,
"gzip_filename": "FILE2",
},
},
}
data = await self.exported_data(self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = gzip.decompress(data[filename])
assert compressed == data[filename]
assert result == self.expected
@coroutine_test
async def test_lzma_plugin(self):
filename = self._named_tempfile("lzma_file")
settings = {
"FEEDS": {
filename: {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
},
},
}
data = await self.exported_data(self.items, settings)
try:
lzma.decompress(data[filename])
except lzma.LZMAError:
pytest.fail("Received invalid lzma data.")
@coroutine_test
async def test_lzma_plugin_format(self):
filename_to_compressed = {
self._named_tempfile("format_FORMAT_XZ"): lzma.compress(
self.expected, format=lzma.FORMAT_XZ
),
self._named_tempfile("format_FORMAT_ALONE"): lzma.compress(
self.expected, format=lzma.FORMAT_ALONE
),
}
settings = {
"FEEDS": {
self._named_tempfile("format_FORMAT_XZ"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_format": lzma.FORMAT_XZ,
},
self._named_tempfile("format_FORMAT_ALONE"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_format": lzma.FORMAT_ALONE,
},
},
}
data = await self.exported_data(self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = lzma.decompress(data[filename])
assert compressed == data[filename]
assert result == self.expected
@coroutine_test
async def test_lzma_plugin_check(self):
filename_to_compressed = {
self._named_tempfile("check_CHECK_NONE"): lzma.compress(
self.expected, check=lzma.CHECK_NONE
),
self._named_tempfile("check_CHECK_CRC256"): lzma.compress(
self.expected, check=lzma.CHECK_SHA256
),
}
settings = {
"FEEDS": {
self._named_tempfile("check_CHECK_NONE"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_check": lzma.CHECK_NONE,
},
self._named_tempfile("check_CHECK_CRC256"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_check": lzma.CHECK_SHA256,
},
},
}
data = await self.exported_data(self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = lzma.decompress(data[filename])
assert compressed == data[filename]
assert result == self.expected
@coroutine_test
async def test_lzma_plugin_preset(self):
filename_to_compressed = {
self._named_tempfile("preset_PRESET_0"): lzma.compress(
self.expected, preset=0
),
self._named_tempfile("preset_PRESET_9"): lzma.compress(
self.expected, preset=9
),
}
settings = {
"FEEDS": {
self._named_tempfile("preset_PRESET_0"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_preset": 0,
},
self._named_tempfile("preset_PRESET_9"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_preset": 9,
},
},
}
data = await self.exported_data(self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = lzma.decompress(data[filename])
assert compressed == data[filename]
assert result == self.expected
@coroutine_test
async def test_lzma_plugin_filters(self):
if "PyPy" in sys.version:
# https://foss.heptapod.net/pypy/pypy/-/issues/3527
pytest.skip("lzma filters doesn't work in PyPy")
filters = [{"id": lzma.FILTER_LZMA2}]
compressed = lzma.compress(self.expected, filters=filters)
filename = self._named_tempfile("filters")
settings = {
"FEEDS": {
filename: {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_filters": filters,
},
},
}
data = await self.exported_data(self.items, settings)
assert compressed == data[filename]
result = lzma.decompress(data[filename])
assert result == self.expected
@coroutine_test
async def test_bz2_plugin(self):
filename = self._named_tempfile("bz2_file")
settings = {
"FEEDS": {
filename: {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"],
},
},
}
data = await self.exported_data(self.items, settings)
try:
bz2.decompress(data[filename])
except OSError:
pytest.fail("Received invalid bz2 data.")
@coroutine_test
async def test_bz2_plugin_compresslevel(self):
filename_to_compressed = {
self._named_tempfile("compresslevel_1"): bz2.compress(
self.expected, compresslevel=1
),
self._named_tempfile("compresslevel_9"): bz2.compress(
self.expected, compresslevel=9
),
}
settings = {
"FEEDS": {
self._named_tempfile("compresslevel_1"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"],
"bz2_compresslevel": 1,
},
self._named_tempfile("compresslevel_9"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"],
"bz2_compresslevel": 9,
},
},
}
data = await self.exported_data(self.items, settings)
for filename, compressed in filename_to_compressed.items():
result = bz2.decompress(data[filename])
assert compressed == data[filename]
assert result == self.expected
@coroutine_test
async def test_custom_plugin(self):
filename = self._named_tempfile("csv_file")
settings = {
"FEEDS": {
filename: {
"format": "csv",
"postprocessing": [self.MyPlugin1],
},
},
}
data = await self.exported_data(self.items, settings)
assert data[filename] == self.expected
@coroutine_test
async def test_custom_plugin_with_parameter(self):
expected = b"foo\r\n\nbar\r\n\n"
filename = self._named_tempfile("newline")
settings = {
"FEEDS": {
filename: {
"format": "csv",
"postprocessing": [self.MyPlugin1],
"plugin1_char": b"\n",
},
},
}
data = await self.exported_data(self.items, settings)
assert data[filename] == expected
@coroutine_test
async def test_custom_plugin_with_compression(self):
expected = b"foo\r\n\nbar\r\n\n"
filename_to_decompressor = {
self._named_tempfile("bz2"): bz2.decompress,
self._named_tempfile("lzma"): lzma.decompress,
self._named_tempfile("gzip"): gzip.decompress,
}
settings = {
"FEEDS": {
self._named_tempfile("bz2"): {
"format": "csv",
"postprocessing": [
self.MyPlugin1,
"scrapy.extensions.postprocessing.Bz2Plugin",
],
"plugin1_char": b"\n",
},
self._named_tempfile("lzma"): {
"format": "csv",
"postprocessing": [
self.MyPlugin1,
"scrapy.extensions.postprocessing.LZMAPlugin",
],
"plugin1_char": b"\n",
},
self._named_tempfile("gzip"): {
"format": "csv",
"postprocessing": [
self.MyPlugin1,
"scrapy.extensions.postprocessing.GzipPlugin",
],
"plugin1_char": b"\n",
},
},
}
data = await self.exported_data(self.items, settings)
for filename, decompressor in filename_to_decompressor.items():
result = decompressor(data[filename])
assert result == expected
@coroutine_test
async def test_exports_compatibility_with_postproc(self):
filename_to_expected = {
self._named_tempfile("csv"): b"foo\r\nbar\r\n",
self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]',
self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n',
self._named_tempfile("xml"): b'<?xml version="1.0" encoding="utf-8"?>\n'
b"<items>\n<item><foo>bar</foo></item>\n</items>",
}
settings = {
"FEEDS": {
self._named_tempfile("csv"): {
"format": "csv",
"postprocessing": [self.MyPlugin1],
# empty plugin to activate postprocessing.PostProcessingManager
},
self._named_tempfile("json"): {
"format": "json",
"postprocessing": [self.MyPlugin1],
},
self._named_tempfile("jsonlines"): {
"format": "jsonlines",
"postprocessing": [self.MyPlugin1],
},
self._named_tempfile("xml"): {
"format": "xml",
"postprocessing": [self.MyPlugin1],
},
self._named_tempfile("marshal"): {
"format": "marshal",
"postprocessing": [self.MyPlugin1],
},
self._named_tempfile("pickle"): {
"format": "pickle",
"postprocessing": [self.MyPlugin1],
},
},
}
data = await self.exported_data(self.items, settings)
for filename, result in data.items():
if "pickle" in filename:
expected, result = self.items[0], pickle.loads(result)
elif "marshal" in filename:
expected, result = self.items[0], marshal.loads(result)
else:
expected = filename_to_expected[filename]
assert result == expected

558
tests/test_feedexport_storages.py Normal file
View File

@ -0,0 +1,558 @@
from __future__ import annotations
import os
import string
import tempfile
from io import BytesIO
from pathlib import Path
from typing import IO, Any
from unittest import mock
from urllib.parse import quote
import pytest
from testfixtures import LogCapture
from w3lib.url import path_to_file_uri
from zope.interface.verify import verifyObject
import scrapy
from scrapy.extensions.feedexport import (
BlockingFeedStorage,
FileFeedStorage,
FTPFeedStorage,
GCSFeedStorage,
IFeedStorage,
S3FeedStorage,
StdoutFeedStorage,
)
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
from tests.utils.decorators import coroutine_test
def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
classes and set their proper return values.
"""
from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415
client_mock = mock.create_autospec(Client)
bucket_mock = mock.create_autospec(Bucket)
client_mock.get_bucket.return_value = bucket_mock
blob_mock = mock.create_autospec(Blob)
bucket_mock.blob.return_value = blob_mock
return (client_mock, bucket_mock, blob_mock)
class TestFileFeedStorage:
def test_store_file_uri(self, tmp_path):
path = tmp_path / "file.txt"
uri = path_to_file_uri(str(path))
self._assert_stores(FileFeedStorage(uri), path)
def test_store_file_uri_makedirs(self, tmp_path):
path = tmp_path / "more" / "paths" / "file.txt"
uri = path_to_file_uri(str(path))
self._assert_stores(FileFeedStorage(uri), path)
def test_store_direct_path(self, tmp_path):
path = tmp_path / "file.txt"
self._assert_stores(FileFeedStorage(str(path)), path)
def test_store_direct_path_relative(self, tmp_path):
old_cwd = Path.cwd()
try:
os.chdir(tmp_path)
path = Path("foo", "bar")
self._assert_stores(FileFeedStorage(str(path)), path)
finally:
os.chdir(old_cwd)
def test_interface(self, tmp_path):
path = tmp_path / "file.txt"
st = FileFeedStorage(str(path))
verifyObject(IFeedStorage, st)
@staticmethod
def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None:
storage = FileFeedStorage(str(path), feed_options=feed_options)
spider = scrapy.Spider("default")
file = storage.open(spider)
file.write(b"content")
storage.store(file)
def test_append(self, tmp_path):
path = tmp_path / "file.txt"
self._store(path)
self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent")
def test_overwrite(self, tmp_path):
path = tmp_path / "file.txt"
self._store(path, {"overwrite": True})
self._assert_stores(
FileFeedStorage(str(path), feed_options={"overwrite": True}), path
)
@staticmethod
def _assert_stores(
storage: FileFeedStorage, path: Path, expected_content: bytes = b"content"
) -> None:
spider = scrapy.Spider("default")
file = storage.open(spider)
file.write(b"content")
storage.store(file)
assert path.exists()
try:
assert path.read_bytes() == expected_content
finally:
path.unlink()
def test_preserves_windows_path_without_file_scheme(self):
path = r"C:\Users\user\Desktop\test.txt"
storage = FileFeedStorage(path)
assert storage.path == path
@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage
class TestFTPFeedStorage:
def get_test_spider(self, settings=None):
class TestSpider(scrapy.Spider):
name = "test_spider"
crawler = get_crawler(settings_dict=settings)
return TestSpider.from_crawler(crawler)
async def _store(self, uri, content, feed_options=None, settings=None):
crawler = get_crawler(settings_dict=settings or {})
storage = FTPFeedStorage.from_crawler(
crawler,
uri,
feed_options=feed_options,
)
verifyObject(IFeedStorage, storage)
spider = self.get_test_spider()
file = storage.open(spider)
file.write(content)
await maybe_deferred_to_future(storage.store(file))
def _assert_stored(self, path: Path, content):
assert path.exists()
try:
assert path.read_bytes() == content
finally:
path.unlink()
@coroutine_test
async def test_append(self):
with MockFTPServer() as ftp_server:
filename = "file"
url = ftp_server.url(filename)
feed_options = {"overwrite": False}
await self._store(url, b"foo", feed_options=feed_options)
await self._store(url, b"bar", feed_options=feed_options)
self._assert_stored(ftp_server.path / filename, b"foobar")
@coroutine_test
async def test_overwrite(self):
with MockFTPServer() as ftp_server:
filename = "file"
url = ftp_server.url(filename)
await self._store(url, b"foo")
await self._store(url, b"bar")
self._assert_stored(ftp_server.path / filename, b"bar")
@coroutine_test
async def test_append_active_mode(self):
with MockFTPServer() as ftp_server:
settings = {"FEED_STORAGE_FTP_ACTIVE": True}
filename = "file"
url = ftp_server.url(filename)
feed_options = {"overwrite": False}
await self._store(url, b"foo", feed_options=feed_options, settings=settings)
await self._store(url, b"bar", feed_options=feed_options, settings=settings)
self._assert_stored(ftp_server.path / filename, b"foobar")
@coroutine_test
async def test_overwrite_active_mode(self):
with MockFTPServer() as ftp_server:
settings = {"FEED_STORAGE_FTP_ACTIVE": True}
filename = "file"
url = ftp_server.url(filename)
await self._store(url, b"foo", settings=settings)
await self._store(url, b"bar", settings=settings)
self._assert_stored(ftp_server.path / filename, b"bar")
def test_uri_auth_quote(self):
# RFC3986: 3.2.1. User Information
pw_quoted = quote(string.punctuation, safe="")
st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {})
assert st.password == string.punctuation
class MyBlockingFeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file: IO[bytes]) -> None:
return
class TestBlockingFeedStorage:
def get_test_spider(self, settings=None):
class TestSpider(scrapy.Spider):
name = "test_spider"
crawler = get_crawler(settings_dict=settings)
return TestSpider.from_crawler(crawler)
def test_default_temp_dir(self):
b = MyBlockingFeedStorage()
storage_file = b.open(self.get_test_spider())
storage_dir = Path(storage_file.name).parent
assert str(storage_dir) == tempfile.gettempdir()
def test_temp_file(self, tmp_path):
b = MyBlockingFeedStorage()
spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)})
storage_file = b.open(spider)
storage_dir = Path(storage_file.name).parent
assert storage_dir == tmp_path
def test_invalid_folder(self, tmp_path):
b = MyBlockingFeedStorage()
invalid_path = tmp_path / "invalid_path"
spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)})
with pytest.raises(OSError, match="Not a Directory:"):
b.open(spider=spider)
@pytest.mark.requires_boto3
@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage
class TestS3FeedStorage:
def test_parse_credentials(self):
aws_credentials = {
"AWS_ACCESS_KEY_ID": "settings_key",
"AWS_SECRET_ACCESS_KEY": "settings_secret",
"AWS_SESSION_TOKEN": "settings_token",
}
crawler = get_crawler(settings_dict=aws_credentials)
# Instantiate with crawler
storage = S3FeedStorage.from_crawler(
crawler,
"s3://mybucket/export.csv",
)
assert storage.access_key == "settings_key"
assert storage.secret_key == "settings_secret"
assert storage.session_token == "settings_token"
# Instantiate directly
storage = S3FeedStorage(
"s3://mybucket/export.csv",
aws_credentials["AWS_ACCESS_KEY_ID"],
aws_credentials["AWS_SECRET_ACCESS_KEY"],
session_token=aws_credentials["AWS_SESSION_TOKEN"],
)
assert storage.access_key == "settings_key"
assert storage.secret_key == "settings_secret"
assert storage.session_token == "settings_token"
# URI priority > settings priority
storage = S3FeedStorage(
"s3://uri_key:uri_secret@mybucket/export.csv",
aws_credentials["AWS_ACCESS_KEY_ID"],
aws_credentials["AWS_SECRET_ACCESS_KEY"],
)
assert storage.access_key == "uri_key"
assert storage.secret_key == "uri_secret"
@coroutine_test
async def test_store(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
"AWS_SECRET_ACCESS_KEY": "secret_key",
}
crawler = get_crawler(settings_dict=settings)
bucket = "mybucket"
key = "export.csv"
storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}")
verifyObject(IFeedStorage, storage)
file = mock.MagicMock()
storage.s3_client = mock.MagicMock()
await maybe_deferred_to_future(storage.store(file))
assert storage.s3_client.upload_fileobj.call_args == mock.call(
Bucket=bucket, Key=key, Fileobj=file
)
def test_init_without_acl(self):
storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.acl is None
def test_init_with_acl(self):
storage = S3FeedStorage(
"s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
)
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.acl == "custom-acl"
def test_init_with_endpoint_url(self):
storage = S3FeedStorage(
"s3://mybucket/export.csv",
"access_key",
"secret_key",
endpoint_url="https://example.com",
)
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.endpoint_url == "https://example.com"
def test_init_with_region_name(self):
region_name = "ap-east-1"
storage = S3FeedStorage(
"s3://mybucket/export.csv",
"access_key",
"secret_key",
region_name=region_name,
)
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.region_name == region_name
assert storage.s3_client._client_config.region_name == region_name
def test_from_crawler_without_acl(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
"AWS_SECRET_ACCESS_KEY": "secret_key",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
crawler,
"s3://mybucket/export.csv",
)
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.acl is None
def test_without_endpoint_url(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
"AWS_SECRET_ACCESS_KEY": "secret_key",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
crawler,
"s3://mybucket/export.csv",
)
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.endpoint_url is None
def test_without_region_name(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
"AWS_SECRET_ACCESS_KEY": "secret_key",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
crawler,
"s3://mybucket/export.csv",
)
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.s3_client._client_config.region_name == "us-east-1"
def test_from_crawler_with_acl(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
"AWS_SECRET_ACCESS_KEY": "secret_key",
"FEED_STORAGE_S3_ACL": "custom-acl",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
crawler,
"s3://mybucket/export.csv",
)
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.acl == "custom-acl"
def test_from_crawler_with_endpoint_url(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
"AWS_SECRET_ACCESS_KEY": "secret_key",
"AWS_ENDPOINT_URL": "https://example.com",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.endpoint_url == "https://example.com"
def test_from_crawler_with_region_name(self):
region_name = "ap-east-1"
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
"AWS_SECRET_ACCESS_KEY": "secret_key",
"AWS_REGION_NAME": region_name,
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.region_name == region_name
assert storage.s3_client._client_config.region_name == region_name
@coroutine_test
async def test_store_without_acl(self):
storage = S3FeedStorage(
"s3://mybucket/export.csv",
"access_key",
"secret_key",
)
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.acl is None
storage.s3_client = mock.MagicMock()
await maybe_deferred_to_future(storage.store(BytesIO(b"test file")))
acl = (
storage.s3_client.upload_fileobj.call_args[1]
.get("ExtraArgs", {})
.get("ACL")
)
assert acl is None
@coroutine_test
async def test_store_with_acl(self):
storage = S3FeedStorage(
"s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
)
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.acl == "custom-acl"
storage.s3_client = mock.MagicMock()
await maybe_deferred_to_future(storage.store(BytesIO(b"test file")))
acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
assert acl == "custom-acl"
def test_overwrite_default(self):
with LogCapture() as log:
S3FeedStorage(
"s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
)
assert "S3 does not support appending to files" not in str(log)
def test_overwrite_false(self):
with LogCapture() as log:
S3FeedStorage(
"s3://mybucket/export.csv",
"access_key",
"secret_key",
"custom-acl",
feed_options={"overwrite": False},
)
assert "S3 does not support appending to files" in str(log)
@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage
class TestGCSFeedStorage:
def test_parse_settings(self):
try:
from google.cloud.storage import Client # noqa: F401,PLC0415
except ImportError:
pytest.skip("GCSFeedStorage requires google-cloud-storage")
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
assert storage.project_id == "123"
assert storage.acl == "publicRead"
assert storage.bucket_name == "mybucket"
assert storage.blob_name == "export.csv"
def test_parse_empty_acl(self):
try:
from google.cloud.storage import Client # noqa: F401,PLC0415
except ImportError:
pytest.skip("GCSFeedStorage requires google-cloud-storage")
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
assert storage.acl is None
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
assert storage.acl is None
@coroutine_test
async def test_store(self):
try:
from google.cloud.storage import Client # noqa: F401,PLC0415
except ImportError:
pytest.skip("GCSFeedStorage requires google-cloud-storage")
uri = "gs://mybucket/export.csv"
project_id = "myproject-123"
acl = "publicRead"
(client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
with mock.patch("google.cloud.storage.Client") as m:
m.return_value = client_mock
f = mock.Mock()
storage = GCSFeedStorage(uri, project_id, acl)
await maybe_deferred_to_future(storage.store(f))
f.seek.assert_called_once_with(0)
m.assert_called_once_with(project=project_id)
client_mock.get_bucket.assert_called_once_with("mybucket")
bucket_mock.blob.assert_called_once_with("export.csv")
blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
def test_overwrite_default(self):
with LogCapture() as log:
GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl")
assert "GCS does not support appending to files" not in str(log)
def test_overwrite_false(self):
with LogCapture() as log:
GCSFeedStorage(
"gs://mybucket/export.csv",
"myproject-123",
"custom-acl",
feed_options={"overwrite": False},
)
assert "GCS does not support appending to files" in str(log)
class TestStdoutFeedStorage:
def test_store(self):
out = BytesIO()
storage = StdoutFeedStorage("stdout:", _stdout=out)
file = storage.open(scrapy.Spider("default"))
file.write(b"content")
storage.store(file)
assert out.getvalue() == b"content"
def test_overwrite_default(self):
with LogCapture() as log:
StdoutFeedStorage("stdout:")
assert (
"Standard output (stdout) storage does not support overwriting"
not in str(log)
)
def test_overwrite_true(self):
with LogCapture() as log:
StdoutFeedStorage("stdout:", feed_options={"overwrite": True})
assert "Standard output (stdout) storage does not support overwriting" in str(
log
)

142
tests/test_feedexport_uri_params.py Normal file
View File

@ -0,0 +1,142 @@
from __future__ import annotations
import warnings
from abc import ABC, abstractmethod
import pytest
import scrapy
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.feedexport import FeedExporter
from scrapy.utils.test import get_crawler
class TestURIParams(ABC):
spider_name = "uri_params_spider"
deprecated_options = False
@abstractmethod
def build_settings(self, uri="file:///tmp/foobar", uri_params=None):
raise NotImplementedError
def _crawler_feed_exporter(self, settings):
if self.deprecated_options:
with pytest.warns(
ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
):
crawler = get_crawler(settings_dict=settings)
else:
crawler = get_crawler(settings_dict=settings)
feed_exporter = crawler.get_extension(FeedExporter)
return crawler, feed_exporter
def test_default(self):
settings = self.build_settings(
uri="file:///tmp/%(name)s",
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}"
def test_none(self):
def uri_params(params, spider):
pass
settings = self.build_settings(
uri="file:///tmp/%(name)s",
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
feed_exporter.open_spider(spider)
assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}"
def test_empty_dict(self):
def uri_params(params, spider):
return {}
settings = self.build_settings(
uri="file:///tmp/%(name)s",
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
with pytest.raises(KeyError):
feed_exporter.open_spider(spider)
def test_params_as_is(self):
def uri_params(params, spider):
return params
settings = self.build_settings(
uri="file:///tmp/%(name)s",
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}"
def test_custom_param(self):
def uri_params(params, spider):
return {**params, "foo": self.spider_name}
settings = self.build_settings(
uri="file:///tmp/%(foo)s",
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}"
class TestURIParamsSetting(TestURIParams):
deprecated_options = True
def build_settings(self, uri="file:///tmp/foobar", uri_params=None):
extra_settings = {}
if uri_params:
extra_settings["FEED_URI_PARAMS"] = uri_params
return {
"FEED_URI": uri,
**extra_settings,
}
class TestURIParamsFeedOption(TestURIParams):
deprecated_options = False
def build_settings(self, uri="file:///tmp/foobar", uri_params=None):
options = {
"format": "jl",
}
if uri_params:
options["uri_params"] = uri_params
return {
"FEEDS": {
uri: options,
},
}

1213
tests/test_http_request.py
View File

File diff suppressed because it is too large Load Diff

1027
tests/test_http_request_form.py Normal file
View File

File diff suppressed because it is too large Load Diff

190
tests/test_http_request_json.py Normal file
View File

@ -0,0 +1,190 @@
from __future__ import annotations
import json
import warnings
from unittest import mock
from scrapy.http import JsonRequest
from scrapy.utils.python import to_bytes
from tests.test_http_request import TestRequest
class TestJsonRequest(TestRequest):
request_class = JsonRequest
default_method = "GET"
default_headers = {
b"Content-Type": [b"application/json"],
b"Accept": [b"application/json, text/javascript, */*; q=0.01"],
}
def test_data(self):
r1 = self.request_class(url="http://www.example.com/")
assert r1.body == b""
body = b"body"
r2 = self.request_class(url="http://www.example.com/", body=body)
assert r2.body == body
data = {
"name": "value",
}
r3 = self.request_class(url="http://www.example.com/", data=data)
assert r3.body == to_bytes(json.dumps(data))
# empty data
r4 = self.request_class(url="http://www.example.com/", data=[])
assert r4.body == to_bytes(json.dumps([]))
def test_data_method(self):
# data is not passed
r1 = self.request_class(url="http://www.example.com/")
assert r1.method == "GET"
body = b"body"
r2 = self.request_class(url="http://www.example.com/", body=body)
assert r2.method == "GET"
data = {
"name": "value",
}
r3 = self.request_class(url="http://www.example.com/", data=data)
assert r3.method == "POST"
# method passed explicitly
r4 = self.request_class(url="http://www.example.com/", data=data, method="GET")
assert r4.method == "GET"
r5 = self.request_class(url="http://www.example.com/", data=[])
assert r5.method == "POST"
def test_body_data(self):
"""passing both body and data should result a warning"""
body = b"body"
data = {
"name": "value",
}
with warnings.catch_warnings(record=True) as _warnings:
r5 = self.request_class(url="http://www.example.com/", body=body, data=data)
assert r5.body == body
assert r5.method == "GET"
assert len(_warnings) == 1
assert "data will be ignored" in str(_warnings[0].message)
def test_empty_body_data(self):
"""passing any body value and data should result a warning"""
data = {
"name": "value",
}
with warnings.catch_warnings(record=True) as _warnings:
r6 = self.request_class(url="http://www.example.com/", body=b"", data=data)
assert r6.body == b""
assert r6.method == "GET"
assert len(_warnings) == 1
assert "data will be ignored" in str(_warnings[0].message)
def test_body_none_data(self):
data = {
"name": "value",
}
with warnings.catch_warnings(record=True) as _warnings:
r7 = self.request_class(url="http://www.example.com/", body=None, data=data)
assert r7.body == to_bytes(json.dumps(data))
assert r7.method == "POST"
assert len(_warnings) == 0
def test_body_data_none(self):
with warnings.catch_warnings(record=True) as _warnings:
r8 = self.request_class(url="http://www.example.com/", body=None, data=None)
assert r8.method == "GET"
assert len(_warnings) == 0
def test_dumps_sort_keys(self):
"""Test that sort_keys=True is passed to json.dumps by default"""
data = {
"name": "value",
}
with mock.patch("json.dumps", return_value=b"") as mock_dumps:
self.request_class(url="http://www.example.com/", data=data)
kwargs = mock_dumps.call_args[1]
assert kwargs["sort_keys"] is True
def test_dumps_kwargs(self):
"""Test that dumps_kwargs are passed to json.dumps"""
data = {
"name": "value",
}
dumps_kwargs = {
"ensure_ascii": True,
"allow_nan": True,
}
with mock.patch("json.dumps", return_value=b"") as mock_dumps:
self.request_class(
url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs
)
kwargs = mock_dumps.call_args[1]
assert kwargs["ensure_ascii"] is True
assert kwargs["allow_nan"] is True
def test_replace_data(self):
data1 = {
"name1": "value1",
}
data2 = {
"name2": "value2",
}
r1 = self.request_class(url="http://www.example.com/", data=data1)
r2 = r1.replace(data=data2)
assert r2.body == to_bytes(json.dumps(data2))
def test_replace_sort_keys(self):
"""Test that replace provides sort_keys=True to json.dumps"""
data1 = {
"name1": "value1",
}
data2 = {
"name2": "value2",
}
r1 = self.request_class(url="http://www.example.com/", data=data1)
with mock.patch("json.dumps", return_value=b"") as mock_dumps:
r1.replace(data=data2)
kwargs = mock_dumps.call_args[1]
assert kwargs["sort_keys"] is True
def test_replace_dumps_kwargs(self):
"""Test that dumps_kwargs are provided to json.dumps when replace is called"""
data1 = {
"name1": "value1",
}
data2 = {
"name2": "value2",
}
dumps_kwargs = {
"ensure_ascii": True,
"allow_nan": True,
}
r1 = self.request_class(
url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs
)
with mock.patch("json.dumps", return_value=b"") as mock_dumps:
r1.replace(data=data2)
kwargs = mock_dumps.call_args[1]
assert kwargs["ensure_ascii"] is True
assert kwargs["allow_nan"] is True
def test_replacement_both_body_and_data_warns(self):
"""Test that we get a warning if both body and data are passed"""
body1 = None
body2 = b"body"
data1 = {
"name1": "value1",
}
data2 = {
"name2": "value2",
}
r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1)
with warnings.catch_warnings(record=True) as _warnings:
r1.replace(data=data2, body=body2)
assert "Both body and data passed. data will be ignored" in str(
_warnings[0].message
)

689
tests/test_http_response.py
View File

@ -1,27 +1,15 @@
import codecs
from unittest import mock
import pytest import pytest
from packaging.version import Version as parse_version from packaging.version import Version as parse_version
from w3lib import __version__ as w3lib_version from w3lib import __version__ as w3lib_version
from w3lib.encoding import resolve_encoding from w3lib.encoding import resolve_encoding
from scrapy.exceptions import NotSupported from scrapy.exceptions import NotSupported
from scrapy.http import ( from scrapy.http import Headers, Request, Response
Headers,
HtmlResponse,
Request,
Response,
TextResponse,
XmlResponse,
)
from scrapy.link import Link from scrapy.link import Link
from scrapy.selector import Selector
from scrapy.utils.python import to_unicode
from tests import get_testdata from tests import get_testdata
class TestResponseBase: class TestResponse:
response_class = Response response_class = Response
def test_init(self): def test_init(self):
@ -344,676 +332,3 @@ class TestResponseBase:
def _links_response_no_href(self): def _links_response_no_href(self):
body = get_testdata("link_extractor", "linkextractor_no_href.html") body = get_testdata("link_extractor", "linkextractor_no_href.html")
return self.response_class("http://example.com/index", body=body) return self.response_class("http://example.com/index", body=body)
class TestTextResponse(TestResponseBase):
response_class = TextResponse
def test_replace(self):
super().test_replace()
r1 = self.response_class(
"http://www.example.com", body="hello", encoding="cp852"
)
r2 = r1.replace(url="http://www.example.com/other")
r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
assert isinstance(r2, self.response_class)
assert r2.url == "http://www.example.com/other"
self._assert_response_encoding(r2, "cp852")
assert r3.url == "http://www.example.com/other"
assert r3._declared_encoding() == "latin1"
def test_unicode_url(self):
# instantiate with unicode url without encoding (should set default encoding)
resp = self.response_class("http://www.example.com/")
self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
# make sure urls are converted to str
resp = self.response_class(url="http://www.example.com/", encoding="utf-8")
assert isinstance(resp.url, str)
resp = self.response_class(
url="http://www.example.com/price/\xa3", encoding="utf-8"
)
assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3")
resp = self.response_class(
url="http://www.example.com/price/\xa3", encoding="latin-1"
)
assert resp.url == "http://www.example.com/price/\xa3"
resp = self.response_class(
"http://www.example.com/price/\xa3",
headers={"Content-type": ["text/html; charset=utf-8"]},
)
assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3")
resp = self.response_class(
"http://www.example.com/price/\xa3",
headers={"Content-type": ["text/html; charset=iso-8859-1"]},
)
assert resp.url == "http://www.example.com/price/\xa3"
def test_unicode_body(self):
unicode_string = (
"\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 "
"\u0442\u0435\u043a\u0441\u0442"
)
with pytest.raises(TypeError):
self.response_class("http://www.example.com", body="unicode body")
original_string = unicode_string.encode("cp1251")
r1 = self.response_class(
"http://www.example.com", body=original_string, encoding="cp1251"
)
# check response.text
assert isinstance(r1.text, str)
assert r1.text == unicode_string
def test_encoding(self):
r1 = self.response_class(
"http://www.example.com",
body=b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=utf-8"]},
)
r2 = self.response_class(
"http://www.example.com", encoding="utf-8", body="\xa3"
)
r3 = self.response_class(
"http://www.example.com",
body=b"\xa3",
headers={"Content-type": ["text/html; charset=iso-8859-1"]},
)
r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
r5 = self.response_class(
"http://www.example.com",
body=b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=None"]},
)
r6 = self.response_class(
"http://www.example.com",
body=b"\xa8D",
headers={"Content-type": ["text/html; charset=gb2312"]},
)
r7 = self.response_class(
"http://www.example.com",
body=b"\xa8D",
headers={"Content-type": ["text/html; charset=gbk"]},
)
r8 = self.response_class(
"http://www.example.com",
body=codecs.BOM_UTF8 + b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=cp1251"]},
)
r9 = self.response_class(
"http://www.example.com",
body=b"\x80",
headers={
"Content-type": [b"application/x-download; filename=\x80dummy.txt"]
},
)
assert r1._headers_encoding() == "utf-8"
assert r2._headers_encoding() is None
assert r2._declared_encoding() == "utf-8"
self._assert_response_encoding(r2, "utf-8")
assert r3._headers_encoding() == "cp1252"
assert r3._declared_encoding() == "cp1252"
assert r4._headers_encoding() is None
assert r5._headers_encoding() is None
assert r8._headers_encoding() == "cp1251"
assert r9._headers_encoding() is None
assert r8._declared_encoding() == "utf-8"
assert r9._declared_encoding() is None
self._assert_response_encoding(r5, "utf-8")
self._assert_response_encoding(r8, "utf-8")
self._assert_response_encoding(r9, "cp1252")
assert r4._body_inferred_encoding() is not None
assert r4._body_inferred_encoding() != "ascii"
self._assert_response_values(r1, "utf-8", "\xa3")
self._assert_response_values(r2, "utf-8", "\xa3")
self._assert_response_values(r3, "iso-8859-1", "\xa3")
self._assert_response_values(r6, "gb18030", "\u2015")
self._assert_response_values(r7, "gb18030", "\u2015")
self._assert_response_values(r9, "cp1252", "€")
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
with pytest.raises(TypeError):
self.response_class("http://www.example.com", body="\xa3")
def test_declared_encoding_invalid(self):
"""Check that unknown declared encodings are ignored"""
r = self.response_class(
"http://www.example.com",
headers={"Content-type": ["text/html; charset=UNKNOWN"]},
body=b"\xc2\xa3",
)
assert r._declared_encoding() is None
self._assert_response_values(r, "utf-8", "\xa3")
def test_utf16(self):
"""Test utf-16 because UnicodeDammit is known to have problems with"""
r = self.response_class(
"http://www.example.com",
body=b"\xff\xfeh\x00i\x00",
encoding="utf-16",
)
self._assert_response_values(r, "utf-16", "hi")
def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
r6 = self.response_class(
"http://www.example.com",
headers={"Content-type": ["text/html; charset=utf-8"]},
body=b"\xef\xbb\xbfWORD\xe3\xab",
)
assert r6.encoding == "utf-8"
assert r6.text in {
"WORD\ufffd\ufffd", # w3lib < 1.19.0
"WORD\ufffd", # w3lib >= 1.19.0
}
def test_bom_is_removed_from_body(self):
# Inferring encoding from body also cache decoded body as sideeffect,
# this test tries to ensure that calling response.encoding and
# response.text in indistinct order doesn't affect final
# response.text in indistinct order doesn't affect final
# values for encoding and decoded body.
url = "http://example.com"
body = b"\xef\xbb\xbfWORD"
headers = {"Content-type": ["text/html; charset=utf-8"]}
# Test response without content-type and BOM encoding
response = self.response_class(url, body=body)
assert response.encoding == "utf-8"
assert response.text == "WORD"
response = self.response_class(url, body=body)
assert response.text == "WORD"
assert response.encoding == "utf-8"
# Body caching sideeffect isn't triggered when encoding is declared in
# content-type header but BOM still need to be removed from decoded
# body
response = self.response_class(url, headers=headers, body=body)
assert response.encoding == "utf-8"
assert response.text == "WORD"
response = self.response_class(url, headers=headers, body=body)
assert response.text == "WORD"
assert response.encoding == "utf-8"
def test_replace_wrong_encoding(self):
"""Test invalid chars are replaced properly"""
r = self.response_class(
"http://www.example.com",
encoding="utf-8",
body=b"PREFIX\xe3\xabSUFFIX",
)
# XXX: Policy for replacing invalid chars may suffer minor variations
# but it should always contain the unicode replacement char ('\ufffd')
assert "\ufffd" in r.text, repr(r.text)
assert "PREFIX" in r.text, repr(r.text)
assert "SUFFIX" in r.text, repr(r.text)
# Do not destroy html tags due to encoding bugs
r = self.response_class(
"http://example.com",
encoding="utf-8",
body=b"\xf0<span>value</span>",
)
assert "<span>value</span>" in r.text, repr(r.text)
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
# r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
# assert '\ufffd' in r.text, repr(r.text)
def test_selector(self):
body = b"<html><head><title>Some page</title><body></body></html>"
response = self.response_class("http://www.example.com", body=body)
assert isinstance(response.selector, Selector)
assert response.selector.type == "html"
assert response.selector is response.selector # property is cached
assert response.selector.response is response
assert response.selector.xpath("//title/text()").getall() == ["Some page"]
assert response.selector.css("title::text").getall() == ["Some page"]
assert response.selector.re("Some (.*)</title>") == ["page"]
def test_selector_shortcuts(self):
body = b"<html><head><title>Some page</title><body></body></html>"
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath("//title/text()").getall()
== response.selector.xpath("//title/text()").getall()
)
assert (
response.css("title::text").getall()
== response.selector.css("title::text").getall()
)
def test_selector_shortcuts_kwargs(self):
body = b'<html><head><title>Some page</title><body><p class="content">A nice paragraph.</p></body></html>'
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath(
"normalize-space(//p[@class=$pclass])", pclass="content"
).getall()
== response.xpath('normalize-space(//p[@class="content"])').getall()
)
assert (
response.xpath(
"//title[count(following::p[@class=$pclass])=$pcount]/text()",
pclass="content",
pcount=1,
).getall()
== response.xpath(
'//title[count(following::p[@class="content"])=1]/text()'
).getall()
)
def test_urljoin_with_base_url(self):
"""Test urljoin shortcut which also evaluates base-url through get_base_url()."""
body = b'<html><body><base href="https://example.net"></body></html>'
joined = self.response_class("http://www.example.com", body=body).urljoin(
"/test"
)
absolute = "https://example.net/test"
assert joined == absolute
body = b'<html><body><base href="/elsewhere"></body></html>'
joined = self.response_class("http://www.example.com", body=body).urljoin(
"test"
)
absolute = "http://www.example.com/test"
assert joined == absolute
body = b'<html><body><base href="/elsewhere/"></body></html>'
joined = self.response_class("http://www.example.com", body=body).urljoin(
"test"
)
absolute = "http://www.example.com/elsewhere/test"
assert joined == absolute
def test_follow_selector(self):
resp = self._links_response()
urls = [
"http://example.com/sample2.html",
"http://example.com/sample3.html",
"http://example.com/sample3.html",
"http://example.com/sample3.html",
"http://example.com/sample3.html#foo",
"http://www.google.com/something",
"http://example.com/innertag.html",
]
# select <a> elements
for sellist in [resp.css("a"), resp.xpath("//a")]:
for sel, url in zip(sellist, urls, strict=False):
self._assert_followed_url(sel, url, response=resp)
# select <link> elements
self._assert_followed_url(
Selector(text='<link href="foo"></link>').css("link")[0],
"http://example.com/foo",
response=resp,
)
# href attributes should work
for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]:
for sel, url in zip(sellist, urls, strict=False):
self._assert_followed_url(sel, url, response=resp)
# non-a elements are not supported
with pytest.raises(
ValueError, match="Only <a> and <link> elements are supported"
):
resp.follow(resp.css("div")[0])
def test_follow_selector_list(self):
resp = self._links_response()
with pytest.raises(ValueError, match="SelectorList"):
resp.follow(resp.css("a"))
def test_follow_selector_invalid(self):
resp = self._links_response()
with pytest.raises(ValueError, match="Unsupported"):
resp.follow(resp.xpath("count(//div)")[0])
def test_follow_selector_attribute(self):
resp = self._links_response()
for src in resp.css("img::attr(src)"):
self._assert_followed_url(src, "http://example.com/sample2.jpg")
def test_follow_selector_no_href(self):
resp = self.response_class(
url="http://example.com",
body=b"<html><body><a name=123>click me</a></body></html>",
)
with pytest.raises(ValueError, match="no href"):
resp.follow(resp.css("a")[0])
def test_follow_whitespace_selector(self):
resp = self.response_class(
"http://example.com",
body=b"""<html><body><a href=" foo\n">click me</a></body></html>""",
)
self._assert_followed_url(
resp.css("a")[0], "http://example.com/foo", response=resp
)
self._assert_followed_url(
resp.css("a::attr(href)")[0],
"http://example.com/foo",
response=resp,
)
def test_follow_encoding(self):
resp1 = self.response_class(
"http://example.com",
encoding="utf8",
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(),
)
req = self._assert_followed_url(
resp1.css("a")[0],
"http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82",
response=resp1,
)
assert req.encoding == "utf8"
resp2 = self.response_class(
"http://example.com",
encoding="cp1251",
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(
"cp1251"
),
)
req = self._assert_followed_url(
resp2.css("a")[0],
"http://example.com/foo?%EF%F0%E8%E2%E5%F2",
response=resp2,
)
assert req.encoding == "cp1251"
def test_follow_flags(self):
res = self.response_class("http://example.com/")
fol = res.follow("http://example.com/", flags=["cached", "allowed"])
assert fol.flags == ["cached", "allowed"]
def test_follow_all_flags(self):
re = self.response_class("http://www.example.com/")
urls = [
"http://www.example.com/",
"http://www.example.com/2",
"http://www.example.com/foo",
]
fol = re.follow_all(urls, flags=["cached", "allowed"])
for req in fol:
assert req.flags == ["cached", "allowed"]
def test_follow_all_css(self):
expected = [
"http://example.com/sample3.html",
"http://example.com/innertag.html",
]
response = self._links_response()
extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
assert expected == extracted
def test_follow_all_css_skip_invalid(self):
expected = [
"http://example.com/page/1/",
"http://example.com/page/3/",
"http://example.com/page/4/",
]
response = self._links_response_no_href()
extracted1 = [r.url for r in response.follow_all(css=".pagination a")]
assert expected == extracted1
extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))]
assert expected == extracted2
def test_follow_all_xpath(self):
expected = [
"http://example.com/sample3.html",
"http://example.com/innertag.html",
]
response = self._links_response()
extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
assert expected == [r.url for r in extracted]
def test_follow_all_xpath_skip_invalid(self):
expected = [
"http://example.com/page/1/",
"http://example.com/page/3/",
"http://example.com/page/4/",
]
response = self._links_response_no_href()
extracted1 = [
r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')
]
assert expected == extracted1
extracted2 = [
r.url
for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))
]
assert expected == extracted2
def test_follow_all_too_many_arguments(self):
response = self._links_response()
with pytest.raises(
ValueError, match="Please supply exactly one of the following arguments"
):
response.follow_all(
css='a[href*="example.com"]',
xpath='//a[contains(@href, "example.com")]',
)
def test_json_response(self):
json_body = b"""{"ip": "109.187.217.200"}"""
json_response = self.response_class("http://www.example.com", body=json_body)
assert json_response.json() == {"ip": "109.187.217.200"}
text_body = b"""<html><body>text</body></html>"""
text_response = self.response_class("http://www.example.com", body=text_body)
with pytest.raises(
ValueError, match=r"(Expecting value|Unexpected '<'): line 1"
):
text_response.json()
def test_cache_json_response(self):
json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]
for json_body in json_valid_bodies:
json_response = self.response_class(
"http://www.example.com", body=json_body
)
with mock.patch("json.loads") as mock_json:
for _ in range(2):
json_response.json()
mock_json.assert_called_once_with(json_body)
class TestHtmlResponse(TestTextResponse):
response_class = HtmlResponse
def test_html_encoding(self):
body = b"""<html><head><title>Some page</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head><body>Price: \xa3100</body></html>'
"""
r1 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r1, "iso-8859-1", body)
body = b"""<?xml version="1.0" encoding="iso-8859-1"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
Price: \xa3100
"""
r2 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r2, "iso-8859-1", body)
# for conflicting declarations headers must take precedence
body = b"""<html><head><title>Some page</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
</head><body>Price: \xa3100</body></html>'
"""
r3 = self.response_class(
"http://www.example.com",
body=body,
headers={"Content-type": ["text/html; charset=iso-8859-1"]},
)
self._assert_response_values(r3, "iso-8859-1", body)
# make sure replace() preserves the encoding of the original response
body = b"New body \xa3"
r4 = r3.replace(body=body)
self._assert_response_values(r4, "iso-8859-1", body)
def test_html5_meta_charset(self):
body = b"""<html><head><meta charset="gb2312" /><title>Some page</title><body>bla bla</body>"""
r1 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r1, "gb2312", body)
class TestXmlResponse(TestTextResponse):
response_class = XmlResponse
def test_xml_encoding(self):
body = b"<xml></xml>"
r1 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body)
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
r2 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r2, "iso-8859-1", body)
# make sure replace() preserves the explicit encoding passed in the __init__ method
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8")
body2 = b"New body"
r4 = r3.replace(body=body2)
self._assert_response_values(r4, "utf-8", body2)
def test_replace_encoding(self):
# make sure replace() keeps the previous encoding unless overridden explicitly
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
body2 = b"""<?xml version="1.0" encoding="utf-8"?><xml></xml>"""
r5 = self.response_class("http://www.example.com", body=body)
r6 = r5.replace(body=body2)
r7 = r5.replace(body=body2, encoding="utf-8")
self._assert_response_values(r5, "iso-8859-1", body)
self._assert_response_values(r6, "iso-8859-1", body2)
self._assert_response_values(r7, "utf-8", body2)
def test_selector(self):
body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
response = self.response_class("http://www.example.com", body=body)
assert isinstance(response.selector, Selector)
assert response.selector.type == "xml"
assert response.selector is response.selector # property is cached
assert response.selector.response is response
assert response.selector.xpath("//elem/text()").getall() == ["value"]
def test_selector_shortcuts(self):
body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath("//elem/text()").getall()
== response.selector.xpath("//elem/text()").getall()
)
def test_selector_shortcuts_kwargs(self):
body = b"""<?xml version="1.0" encoding="utf-8"?>
<xml xmlns:somens="http://scrapy.org">
<somens:elem>value</somens:elem>
</xml>"""
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath(
"//s:elem/text()", namespaces={"s": "http://scrapy.org"}
).getall()
== response.selector.xpath(
"//s:elem/text()", namespaces={"s": "http://scrapy.org"}
).getall()
)
response.selector.register_namespace("s2", "http://scrapy.org")
assert (
response.xpath(
"//s1:elem/text()", namespaces={"s1": "http://scrapy.org"}
).getall()
== response.selector.xpath("//s2:elem/text()").getall()
)
class CustomResponse(TextResponse):
attributes = (*TextResponse.attributes, "foo", "bar")
def __init__(self, *args, **kwargs) -> None:
self.foo = kwargs.pop("foo", None)
self.bar = kwargs.pop("bar", None)
self.lost = kwargs.pop("lost", None)
super().__init__(*args, **kwargs)
class TestCustomResponse(TestTextResponse):
response_class = CustomResponse
def test_copy(self):
super().test_copy()
r1 = self.response_class(
url="https://example.org",
status=200,
foo="foo",
bar="bar",
lost="lost",
)
r2 = r1.copy()
assert isinstance(r2, self.response_class)
assert r1.foo == r2.foo
assert r1.bar == r2.bar
assert r1.lost == "lost"
assert r2.lost is None
def test_replace(self):
super().test_replace()
r1 = self.response_class(
url="https://example.org",
status=200,
foo="foo",
bar="bar",
lost="lost",
)
r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost")
assert isinstance(r2, self.response_class)
assert r1.foo == "foo"
assert r1.bar == "bar"
assert r1.lost == "lost"
assert r2.foo == "new-foo"
assert r2.bar == "new-bar"
assert r2.lost == "new-lost"
r3 = r1.replace(foo="new-foo", bar="new-bar")
assert isinstance(r3, self.response_class)
assert r1.foo == "foo"
assert r1.bar == "bar"
assert r1.lost == "lost"
assert r3.foo == "new-foo"
assert r3.bar == "new-bar"
assert r3.lost is None
r4 = r1.replace(foo="new-foo")
assert isinstance(r4, self.response_class)
assert r1.foo == "foo"
assert r1.bar == "bar"
assert r1.lost == "lost"
assert r4.foo == "new-foo"
assert r4.bar == "bar"
assert r4.lost is None
with pytest.raises(
TypeError,
match=r"__init__\(\) got an unexpected keyword argument 'unknown'",
):
r1.replace(unknown="unknown")

684
tests/test_http_response_text.py Normal file
View File

@ -0,0 +1,684 @@
from __future__ import annotations
import codecs
from unittest import mock
import pytest
from scrapy.http import HtmlResponse, TextResponse, XmlResponse
from scrapy.selector import Selector
from scrapy.utils.python import to_unicode
from tests.test_http_response import TestResponse
class TestTextResponse(TestResponse):
response_class = TextResponse
def test_replace(self):
super().test_replace()
r1 = self.response_class(
"http://www.example.com", body="hello", encoding="cp852"
)
r2 = r1.replace(url="http://www.example.com/other")
r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
assert isinstance(r2, self.response_class)
assert r2.url == "http://www.example.com/other"
self._assert_response_encoding(r2, "cp852")
assert r3.url == "http://www.example.com/other"
assert r3._declared_encoding() == "latin1"
def test_unicode_url(self):
# instantiate with unicode url without encoding (should set default encoding)
resp = self.response_class("http://www.example.com/")
self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
# make sure urls are converted to str
resp = self.response_class(url="http://www.example.com/", encoding="utf-8")
assert isinstance(resp.url, str)
resp = self.response_class(
url="http://www.example.com/price/\xa3", encoding="utf-8"
)
assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3")
resp = self.response_class(
url="http://www.example.com/price/\xa3", encoding="latin-1"
)
assert resp.url == "http://www.example.com/price/\xa3"
resp = self.response_class(
"http://www.example.com/price/\xa3",
headers={"Content-type": ["text/html; charset=utf-8"]},
)
assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3")
resp = self.response_class(
"http://www.example.com/price/\xa3",
headers={"Content-type": ["text/html; charset=iso-8859-1"]},
)
assert resp.url == "http://www.example.com/price/\xa3"
def test_unicode_body(self):
unicode_string = (
"\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 "
"\u0442\u0435\u043a\u0441\u0442"
)
with pytest.raises(TypeError):
self.response_class("http://www.example.com", body="unicode body")
original_string = unicode_string.encode("cp1251")
r1 = self.response_class(
"http://www.example.com", body=original_string, encoding="cp1251"
)
# check response.text
assert isinstance(r1.text, str)
assert r1.text == unicode_string
def test_encoding(self):
r1 = self.response_class(
"http://www.example.com",
body=b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=utf-8"]},
)
r2 = self.response_class(
"http://www.example.com", encoding="utf-8", body="\xa3"
)
r3 = self.response_class(
"http://www.example.com",
body=b"\xa3",
headers={"Content-type": ["text/html; charset=iso-8859-1"]},
)
r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
r5 = self.response_class(
"http://www.example.com",
body=b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=None"]},
)
r6 = self.response_class(
"http://www.example.com",
body=b"\xa8D",
headers={"Content-type": ["text/html; charset=gb2312"]},
)
r7 = self.response_class(
"http://www.example.com",
body=b"\xa8D",
headers={"Content-type": ["text/html; charset=gbk"]},
)
r8 = self.response_class(
"http://www.example.com",
body=codecs.BOM_UTF8 + b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=cp1251"]},
)
r9 = self.response_class(
"http://www.example.com",
body=b"\x80",
headers={
"Content-type": [b"application/x-download; filename=\x80dummy.txt"]
},
)
assert r1._headers_encoding() == "utf-8"
assert r2._headers_encoding() is None
assert r2._declared_encoding() == "utf-8"
self._assert_response_encoding(r2, "utf-8")
assert r3._headers_encoding() == "cp1252"
assert r3._declared_encoding() == "cp1252"
assert r4._headers_encoding() is None
assert r5._headers_encoding() is None
assert r8._headers_encoding() == "cp1251"
assert r9._headers_encoding() is None
assert r8._declared_encoding() == "utf-8"
assert r9._declared_encoding() is None
self._assert_response_encoding(r5, "utf-8")
self._assert_response_encoding(r8, "utf-8")
self._assert_response_encoding(r9, "cp1252")
assert r4._body_inferred_encoding() is not None
assert r4._body_inferred_encoding() != "ascii"
self._assert_response_values(r1, "utf-8", "\xa3")
self._assert_response_values(r2, "utf-8", "\xa3")
self._assert_response_values(r3, "iso-8859-1", "\xa3")
self._assert_response_values(r6, "gb18030", "\u2015")
self._assert_response_values(r7, "gb18030", "\u2015")
self._assert_response_values(r9, "cp1252", "€")
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
with pytest.raises(TypeError):
self.response_class("http://www.example.com", body="\xa3")
def test_declared_encoding_invalid(self):
"""Check that unknown declared encodings are ignored"""
r = self.response_class(
"http://www.example.com",
headers={"Content-type": ["text/html; charset=UNKNOWN"]},
body=b"\xc2\xa3",
)
assert r._declared_encoding() is None
self._assert_response_values(r, "utf-8", "\xa3")
def test_utf16(self):
"""Test utf-16 because UnicodeDammit is known to have problems with"""
r = self.response_class(
"http://www.example.com",
body=b"\xff\xfeh\x00i\x00",
encoding="utf-16",
)
self._assert_response_values(r, "utf-16", "hi")
def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
r6 = self.response_class(
"http://www.example.com",
headers={"Content-type": ["text/html; charset=utf-8"]},
body=b"\xef\xbb\xbfWORD\xe3\xab",
)
assert r6.encoding == "utf-8"
assert r6.text in {
"WORD\ufffd\ufffd", # w3lib < 1.19.0
"WORD\ufffd", # w3lib >= 1.19.0
}
def test_bom_is_removed_from_body(self):
# Inferring encoding from body also cache decoded body as sideeffect,
# this test tries to ensure that calling response.encoding and
# response.text in indistinct order doesn't affect final
# response.text in indistinct order doesn't affect final
# values for encoding and decoded body.
url = "http://example.com"
body = b"\xef\xbb\xbfWORD"
headers = {"Content-type": ["text/html; charset=utf-8"]}
# Test response without content-type and BOM encoding
response = self.response_class(url, body=body)
assert response.encoding == "utf-8"
assert response.text == "WORD"
response = self.response_class(url, body=body)
assert response.text == "WORD"
assert response.encoding == "utf-8"
# Body caching sideeffect isn't triggered when encoding is declared in
# content-type header but BOM still need to be removed from decoded
# body
response = self.response_class(url, headers=headers, body=body)
assert response.encoding == "utf-8"
assert response.text == "WORD"
response = self.response_class(url, headers=headers, body=body)
assert response.text == "WORD"
assert response.encoding == "utf-8"
def test_replace_wrong_encoding(self):
"""Test invalid chars are replaced properly"""
r = self.response_class(
"http://www.example.com",
encoding="utf-8",
body=b"PREFIX\xe3\xabSUFFIX",
)
# XXX: Policy for replacing invalid chars may suffer minor variations
# but it should always contain the unicode replacement char ('\ufffd')
assert "\ufffd" in r.text, repr(r.text)
assert "PREFIX" in r.text, repr(r.text)
assert "SUFFIX" in r.text, repr(r.text)
# Do not destroy html tags due to encoding bugs
r = self.response_class(
"http://example.com",
encoding="utf-8",
body=b"\xf0<span>value</span>",
)
assert "<span>value</span>" in r.text, repr(r.text)
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
# r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
# assert '\ufffd' in r.text, repr(r.text)
def test_selector(self):
body = b"<html><head><title>Some page</title><body></body></html>"
response = self.response_class("http://www.example.com", body=body)
assert isinstance(response.selector, Selector)
assert response.selector.type == "html"
assert response.selector is response.selector # property is cached
assert response.selector.response is response
assert response.selector.xpath("//title/text()").getall() == ["Some page"]
assert response.selector.css("title::text").getall() == ["Some page"]
assert response.selector.re("Some (.*)</title>") == ["page"]
def test_selector_shortcuts(self):
body = b"<html><head><title>Some page</title><body></body></html>"
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath("//title/text()").getall()
== response.selector.xpath("//title/text()").getall()
)
assert (
response.css("title::text").getall()
== response.selector.css("title::text").getall()
)
def test_selector_shortcuts_kwargs(self):
body = b'<html><head><title>Some page</title><body><p class="content">A nice paragraph.</p></body></html>'
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath(
"normalize-space(//p[@class=$pclass])", pclass="content"
).getall()
== response.xpath('normalize-space(//p[@class="content"])').getall()
)
assert (
response.xpath(
"//title[count(following::p[@class=$pclass])=$pcount]/text()",
pclass="content",
pcount=1,
).getall()
== response.xpath(
'//title[count(following::p[@class="content"])=1]/text()'
).getall()
)
def test_urljoin_with_base_url(self):
"""Test urljoin shortcut which also evaluates base-url through get_base_url()."""
body = b'<html><body><base href="https://example.net"></body></html>'
joined = self.response_class("http://www.example.com", body=body).urljoin(
"/test"
)
absolute = "https://example.net/test"
assert joined == absolute
body = b'<html><body><base href="/elsewhere"></body></html>'
joined = self.response_class("http://www.example.com", body=body).urljoin(
"test"
)
absolute = "http://www.example.com/test"
assert joined == absolute
body = b'<html><body><base href="/elsewhere/"></body></html>'
joined = self.response_class("http://www.example.com", body=body).urljoin(
"test"
)
absolute = "http://www.example.com/elsewhere/test"
assert joined == absolute
def test_follow_selector(self):
resp = self._links_response()
urls = [
"http://example.com/sample2.html",
"http://example.com/sample3.html",
"http://example.com/sample3.html",
"http://example.com/sample3.html",
"http://example.com/sample3.html#foo",
"http://www.google.com/something",
"http://example.com/innertag.html",
]
# select <a> elements
for sellist in [resp.css("a"), resp.xpath("//a")]:
for sel, url in zip(sellist, urls, strict=False):
self._assert_followed_url(sel, url, response=resp)
# select <link> elements
self._assert_followed_url(
Selector(text='<link href="foo"></link>').css("link")[0],
"http://example.com/foo",
response=resp,
)
# href attributes should work
for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]:
for sel, url in zip(sellist, urls, strict=False):
self._assert_followed_url(sel, url, response=resp)
# non-a elements are not supported
with pytest.raises(
ValueError, match="Only <a> and <link> elements are supported"
):
resp.follow(resp.css("div")[0])
def test_follow_selector_list(self):
resp = self._links_response()
with pytest.raises(ValueError, match="SelectorList"):
resp.follow(resp.css("a"))
def test_follow_selector_invalid(self):
resp = self._links_response()
with pytest.raises(ValueError, match="Unsupported"):
resp.follow(resp.xpath("count(//div)")[0])
def test_follow_selector_attribute(self):
resp = self._links_response()
for src in resp.css("img::attr(src)"):
self._assert_followed_url(src, "http://example.com/sample2.jpg")
def test_follow_selector_no_href(self):
resp = self.response_class(
url="http://example.com",
body=b"<html><body><a name=123>click me</a></body></html>",
)
with pytest.raises(ValueError, match="no href"):
resp.follow(resp.css("a")[0])
def test_follow_whitespace_selector(self):
resp = self.response_class(
"http://example.com",
body=b"""<html><body><a href=" foo\n">click me</a></body></html>""",
)
self._assert_followed_url(
resp.css("a")[0], "http://example.com/foo", response=resp
)
self._assert_followed_url(
resp.css("a::attr(href)")[0],
"http://example.com/foo",
response=resp,
)
def test_follow_encoding(self):
resp1 = self.response_class(
"http://example.com",
encoding="utf8",
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(),
)
req = self._assert_followed_url(
resp1.css("a")[0],
"http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82",
response=resp1,
)
assert req.encoding == "utf8"
resp2 = self.response_class(
"http://example.com",
encoding="cp1251",
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(
"cp1251"
),
)
req = self._assert_followed_url(
resp2.css("a")[0],
"http://example.com/foo?%EF%F0%E8%E2%E5%F2",
response=resp2,
)
assert req.encoding == "cp1251"
def test_follow_flags(self):
res = self.response_class("http://example.com/")
fol = res.follow("http://example.com/", flags=["cached", "allowed"])
assert fol.flags == ["cached", "allowed"]
def test_follow_all_flags(self):
re = self.response_class("http://www.example.com/")
urls = [
"http://www.example.com/",
"http://www.example.com/2",
"http://www.example.com/foo",
]
fol = re.follow_all(urls, flags=["cached", "allowed"])
for req in fol:
assert req.flags == ["cached", "allowed"]
def test_follow_all_css(self):
expected = [
"http://example.com/sample3.html",
"http://example.com/innertag.html",
]
response = self._links_response()
extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]
assert expected == extracted
def test_follow_all_css_skip_invalid(self):
expected = [
"http://example.com/page/1/",
"http://example.com/page/3/",
"http://example.com/page/4/",
]
response = self._links_response_no_href()
extracted1 = [r.url for r in response.follow_all(css=".pagination a")]
assert expected == extracted1
extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))]
assert expected == extracted2
def test_follow_all_xpath(self):
expected = [
"http://example.com/sample3.html",
"http://example.com/innertag.html",
]
response = self._links_response()
extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')
assert expected == [r.url for r in extracted]
def test_follow_all_xpath_skip_invalid(self):
expected = [
"http://example.com/page/1/",
"http://example.com/page/3/",
"http://example.com/page/4/",
]
response = self._links_response_no_href()
extracted1 = [
r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')
]
assert expected == extracted1
extracted2 = [
r.url
for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))
]
assert expected == extracted2
def test_follow_all_too_many_arguments(self):
response = self._links_response()
with pytest.raises(
ValueError, match="Please supply exactly one of the following arguments"
):
response.follow_all(
css='a[href*="example.com"]',
xpath='//a[contains(@href, "example.com")]',
)
def test_json_response(self):
json_body = b"""{"ip": "109.187.217.200"}"""
json_response = self.response_class("http://www.example.com", body=json_body)
assert json_response.json() == {"ip": "109.187.217.200"}
text_body = b"""<html><body>text</body></html>"""
text_response = self.response_class("http://www.example.com", body=text_body)
with pytest.raises(
ValueError, match=r"(Expecting value|Unexpected '<'): line 1"
):
text_response.json()
def test_cache_json_response(self):
json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""]
for json_body in json_valid_bodies:
json_response = self.response_class(
"http://www.example.com", body=json_body
)
with mock.patch("json.loads") as mock_json:
for _ in range(2):
json_response.json()
mock_json.assert_called_once_with(json_body)
class TestHtmlResponse(TestTextResponse):
response_class = HtmlResponse
def test_html_encoding(self):
body = b"""<html><head><title>Some page</title>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head><body>Price: \xa3100</body></html>'
"""
r1 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r1, "iso-8859-1", body)
body = b"""<?xml version="1.0" encoding="iso-8859-1"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
Price: \xa3100
"""
r2 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r2, "iso-8859-1", body)
# for conflicting declarations headers must take precedence
body = b"""<html><head><title>Some page</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
</head><body>Price: \xa3100</body></html>'
"""
r3 = self.response_class(
"http://www.example.com",
body=body,
headers={"Content-type": ["text/html; charset=iso-8859-1"]},
)
self._assert_response_values(r3, "iso-8859-1", body)
# make sure replace() preserves the encoding of the original response
body = b"New body \xa3"
r4 = r3.replace(body=body)
self._assert_response_values(r4, "iso-8859-1", body)
def test_html5_meta_charset(self):
body = b"""<html><head><meta charset="gb2312" /><title>Some page</title><body>bla bla</body>"""
r1 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r1, "gb2312", body)
class TestXmlResponse(TestTextResponse):
response_class = XmlResponse
def test_xml_encoding(self):
body = b"<xml></xml>"
r1 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body)
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
r2 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r2, "iso-8859-1", body)
# make sure replace() preserves the explicit encoding passed in the __init__ method
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8")
body2 = b"New body"
r4 = r3.replace(body=body2)
self._assert_response_values(r4, "utf-8", body2)
def test_replace_encoding(self):
# make sure replace() keeps the previous encoding unless overridden explicitly
body = b"""<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
body2 = b"""<?xml version="1.0" encoding="utf-8"?><xml></xml>"""
r5 = self.response_class("http://www.example.com", body=body)
r6 = r5.replace(body=body2)
r7 = r5.replace(body=body2, encoding="utf-8")
self._assert_response_values(r5, "iso-8859-1", body)
self._assert_response_values(r6, "iso-8859-1", body2)
self._assert_response_values(r7, "utf-8", body2)
def test_selector(self):
body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
response = self.response_class("http://www.example.com", body=body)
assert isinstance(response.selector, Selector)
assert response.selector.type == "xml"
assert response.selector is response.selector # property is cached
assert response.selector.response is response
assert response.selector.xpath("//elem/text()").getall() == ["value"]
def test_selector_shortcuts(self):
body = b'<?xml version="1.0" encoding="utf-8"?><xml><elem>value</elem></xml>'
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath("//elem/text()").getall()
== response.selector.xpath("//elem/text()").getall()
)
def test_selector_shortcuts_kwargs(self):
body = b"""<?xml version="1.0" encoding="utf-8"?>
<xml xmlns:somens="http://scrapy.org">
<somens:elem>value</somens:elem>
</xml>"""
response = self.response_class("http://www.example.com", body=body)
assert (
response.xpath(
"//s:elem/text()", namespaces={"s": "http://scrapy.org"}
).getall()
== response.selector.xpath(
"//s:elem/text()", namespaces={"s": "http://scrapy.org"}
).getall()
)
response.selector.register_namespace("s2", "http://scrapy.org")
assert (
response.xpath(
"//s1:elem/text()", namespaces={"s1": "http://scrapy.org"}
).getall()
== response.selector.xpath("//s2:elem/text()").getall()
)
class CustomResponse(TextResponse):
attributes = (*TextResponse.attributes, "foo", "bar")
def __init__(self, *args, **kwargs) -> None:
self.foo = kwargs.pop("foo", None)
self.bar = kwargs.pop("bar", None)
self.lost = kwargs.pop("lost", None)
super().__init__(*args, **kwargs)
class TestCustomResponse(TestTextResponse):
response_class = CustomResponse
def test_copy(self):
super().test_copy()
r1 = self.response_class(
url="https://example.org",
status=200,
foo="foo",
bar="bar",
lost="lost",
)
r2 = r1.copy()
assert isinstance(r2, self.response_class)
assert r1.foo == r2.foo
assert r1.bar == r2.bar
assert r1.lost == "lost"
assert r2.lost is None
def test_replace(self):
super().test_replace()
r1 = self.response_class(
url="https://example.org",
status=200,
foo="foo",
bar="bar",
lost="lost",
)
r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost")
assert isinstance(r2, self.response_class)
assert r1.foo == "foo"
assert r1.bar == "bar"
assert r1.lost == "lost"
assert r2.foo == "new-foo"
assert r2.bar == "new-bar"
assert r2.lost == "new-lost"
r3 = r1.replace(foo="new-foo", bar="new-bar")
assert isinstance(r3, self.response_class)
assert r1.foo == "foo"
assert r1.bar == "bar"
assert r1.lost == "lost"
assert r3.foo == "new-foo"
assert r3.bar == "new-bar"
assert r3.lost is None
r4 = r1.replace(foo="new-foo")
assert isinstance(r4, self.response_class)
assert r1.foo == "foo"
assert r1.bar == "bar"
assert r1.lost == "lost"
assert r4.foo == "new-foo"
assert r4.bar == "bar"
assert r4.lost is None
with pytest.raises(
TypeError,
match=r"__init__\(\) got an unexpected keyword argument 'unknown'",
):
r1.replace(unknown="unknown")

645
tests/test_spider.py
View File

@ -1,35 +1,19 @@
from __future__ import annotations from __future__ import annotations
import gzip
import re
import warnings
from datetime import datetime
from io import BytesIO
from logging import ERROR, WARNING
from pathlib import Path
from typing import Any from typing import Any
from unittest import mock from unittest import mock
import pytest import pytest
from testfixtures import LogCapture from testfixtures import LogCapture
from w3lib.url import safe_url_string
from scrapy import signals from scrapy import signals
from scrapy.crawler import Crawler from scrapy.crawler import Crawler
from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse from scrapy.http import Response, TextResponse, XmlResponse
from scrapy.linkextractors import LinkExtractor
from scrapy.settings import Settings from scrapy.settings import Settings
from scrapy.spiders import ( from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
CrawlSpider,
CSVFeedSpider,
Rule,
SitemapSpider,
Spider,
XMLFeedSpider,
)
from scrapy.spiders.init import InitSpider from scrapy.spiders.init import InitSpider
from scrapy.utils.test import get_crawler, get_reactor_settings from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import get_testdata, tests_datadir from tests import get_testdata
from tests.utils.decorators import coroutine_test, inline_callbacks_test from tests.utils.decorators import coroutine_test, inline_callbacks_test
@ -229,629 +213,6 @@ class TestCSVFeedSpider(TestSpider):
assert len(rows) == 4 assert len(rows) == 4
class TestCrawlSpider(TestSpider):
test_body = b"""<html><head><title>Page title</title></head>
<body>
<p><a href="item/12.html">Item 12</a></p>
<div class='links'>
<p><a href="/about.html">About us</a></p>
</div>
<div>
<p><a href="/nofollow.html">This shouldn't be followed</a></p>
</div>
</body></html>"""
spider_class = CrawlSpider
def test_rule_without_link_extractor(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(),)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
def test_process_links(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
def dummy_process_links(self, links):
return links
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
def test_process_links_filter(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
_test_regex = re.compile("nofollow")
def filter_process_links(self, links):
return [link for link in links if not self._test_regex.search(link.url)]
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 2
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
]
def test_process_links_generator(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
def dummy_process_links(self, links):
yield from links
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
def test_process_request(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
def process_request_change_domain(request, response):
return request.replace(url=request.url.replace(".org", ".com"))
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (
Rule(LinkExtractor(), process_request=process_request_change_domain),
)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.com/somepage/item/12.html",
"http://example.com/about.html",
"http://example.com/nofollow.html",
]
def test_process_request_with_response(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
def process_request_meta_response_class(request, response):
request.meta["response_class"] = response.__class__.__name__
return request
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (
Rule(
LinkExtractor(), process_request=process_request_meta_response_class
),
)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
assert [r.meta["response_class"] for r in output] == [
"HtmlResponse",
"HtmlResponse",
"HtmlResponse",
]
def test_process_request_instance_method(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_request="process_request_upper"),)
def process_request_upper(self, request, response):
return request.replace(url=request.url.upper())
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),
safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),
safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),
]
def test_process_request_instance_method_with_response(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (
Rule(
LinkExtractor(),
process_request="process_request_meta_response_class",
),
)
def process_request_meta_response_class(self, request, response):
request.meta["response_class"] = response.__class__.__name__
return request
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
assert [r.meta["response_class"] for r in output] == [
"HtmlResponse",
"HtmlResponse",
"HtmlResponse",
]
def test_follow_links_attribute_population(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
assert hasattr(spider, "_follow_links")
assert spider._follow_links
settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False}
crawler = get_crawler(settings_dict=settings_dict)
spider = self.spider_class.from_crawler(crawler, "example.com")
assert hasattr(spider, "_follow_links")
assert not spider._follow_links
@inline_callbacks_test
def test_start_url(self):
class TestSpider(self.spider_class):
name = "test"
start_url = "https://www.example.com"
crawler = get_crawler(TestSpider)
with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log:
yield crawler.crawl()
assert "Error while reading start items and requests" in str(log)
assert "did you miss an 's'?" in str(log)
def test_parse_response_use(self):
class _CrawlSpider(CrawlSpider):
name = "test"
start_urls = "https://www.example.com"
_follow_links = False
with warnings.catch_warnings(record=True) as w:
spider = _CrawlSpider()
assert len(w) == 0
spider._parse_response(
TextResponse(spider.start_urls, body=b""), None, None
)
assert len(w) == 1
def test_parse_response_override(self):
class _CrawlSpider(CrawlSpider):
def _parse_response(self, response, callback, cb_kwargs, follow=True):
pass
name = "test"
start_urls = "https://www.example.com"
_follow_links = False
with warnings.catch_warnings(record=True) as w:
assert len(w) == 0
spider = _CrawlSpider()
assert len(w) == 1
spider._parse_response(
TextResponse(spider.start_urls, body=b""), None, None
)
assert len(w) == 1
def test_parse_with_rules(self):
class _CrawlSpider(CrawlSpider):
name = "test"
start_urls = "https://www.example.com"
with warnings.catch_warnings(record=True) as w:
spider = _CrawlSpider()
spider.parse_with_rules(
TextResponse(spider.start_urls, body=b""), None, None
)
assert len(w) == 0
class TestSitemapSpider(TestSpider):
spider_class = SitemapSpider
BODY = b"SITEMAP"
f = BytesIO()
g = gzip.GzipFile(fileobj=f, mode="w+b")
g.write(BODY)
g.close()
GZBODY = f.getvalue()
def assertSitemapBody(self, response: Response, body: bytes | None) -> None:
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
assert spider._get_sitemap_body(response) == body
def test_get_sitemap_body(self):
r = XmlResponse(url="http://www.example.com/", body=self.BODY)
self.assertSitemapBody(r, self.BODY)
r = HtmlResponse(url="http://www.example.com/", body=self.BODY)
self.assertSitemapBody(r, None)
r = Response(url="http://www.example.com/favicon.ico", body=self.BODY)
self.assertSitemapBody(r, None)
def test_get_sitemap_body_gzip_headers(self):
r = Response(
url="http://www.example.com/sitemap",
body=self.GZBODY,
headers={"content-type": "application/gzip"},
request=Request("http://www.example.com/sitemap"),
)
self.assertSitemapBody(r, self.BODY)
def test_get_sitemap_body_xml_url(self):
r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY)
self.assertSitemapBody(r, self.BODY)
def test_get_sitemap_body_xml_url_compressed(self):
r = Response(
url="http://www.example.com/sitemap.xml.gz",
body=self.GZBODY,
request=Request("http://www.example.com/sitemap"),
)
self.assertSitemapBody(r, self.BODY)
# .xml.gz but body decoded by HttpCompression middleware already
r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY)
self.assertSitemapBody(r, self.BODY)
def test_get_sitemap_urls_from_robotstxt(self):
robots = b"""# Sitemap files
Sitemap: http://example.com/sitemap.xml
Sitemap: http://example.com/sitemap-product-index.xml
Sitemap: HTTP://example.com/sitemap-uppercase.xml
Sitemap: /sitemap-relative-url.xml
"""
r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://example.com/sitemap.xml",
"http://example.com/sitemap-product-index.xml",
"http://example.com/sitemap-uppercase.xml",
"http://www.example.com/sitemap-relative-url.xml",
]
def test_alternate_url_locs(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>http://www.example.com/english/</loc>
<xhtml:link rel="alternate" hreflang="de"
href="http://www.example.com/deutsch/"/>
<xhtml:link rel="alternate" hreflang="de-ch"
href="http://www.example.com/schweiz-deutsch/"/>
<xhtml:link rel="alternate" hreflang="it"
href="http://www.example.com/italiano/"/>
<xhtml:link rel="alternate" hreflang="it"/><!-- wrong tag without href -->
</url>
</urlset>"""
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/english/"
]
spider.sitemap_alternate_links = True
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/english/",
"http://www.example.com/deutsch/",
"http://www.example.com/schweiz-deutsch/",
"http://www.example.com/italiano/",
]
def test_sitemap_filter(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>http://www.example.com/english/</loc>
<lastmod>2010-01-01</lastmod>
</url>
<url>
<loc>http://www.example.com/portuguese/</loc>
<lastmod>2005-01-01</lastmod>
</url>
</urlset>"""
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
if date_time.year > 2008:
yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/english/",
"http://www.example.com/portuguese/",
]
spider = FilteredSitemapSpider("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/english/"
]
def test_sitemap_filter_with_alternate_links(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>http://www.example.com/english/article_1/</loc>
<lastmod>2010-01-01</lastmod>
<xhtml:link rel="alternate" hreflang="de"
href="http://www.example.com/deutsch/article_1/"/>
</url>
<url>
<loc>http://www.example.com/english/article_2/</loc>
<lastmod>2015-01-01</lastmod>
</url>
</urlset>"""
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
for entry in entries:
alternate_links = entry.get("alternate", ())
for link in alternate_links:
if "/deutsch/" in link:
entry["loc"] = link
yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/english/article_1/",
"http://www.example.com/english/article_2/",
]
spider = FilteredSitemapSpider("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/deutsch/article_1/"
]
def test_sitemapindex_filter(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap>
<loc>http://www.example.com/sitemap1.xml</loc>
<lastmod>2004-01-01T20:00:00+00:00</lastmod>
</sitemap>
<sitemap>
<loc>http://www.example.com/sitemap2.xml</loc>
<lastmod>2005-01-01</lastmod>
</sitemap>
</sitemapindex>"""
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(
entry["lastmod"].split("T")[0], "%Y-%m-%d"
)
if date_time.year > 2004:
yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/sitemap1.xml",
"http://www.example.com/sitemap2.xml",
]
spider = FilteredSitemapSpider("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/sitemap2.xml"
]
def test_compression_bomb_setting(self):
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
assert spider._get_sitemap_body(response) is None
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr(self):
class DownloadMaxSizeSpider(self.spider_class):
download_maxsize = 10_000_000
crawler = get_crawler()
spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
assert spider._get_sitemap_body(response) is None
def test_compression_bomb_request_meta(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_maxsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
assert spider._get_sitemap_body(response) is None
def test_download_warnsize_setting(self):
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_download_warnsize_spider_attr(self):
class DownloadWarnSizeSpider(self.spider_class):
download_warnsize = 10_000_000
crawler = get_crawler()
spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_warnsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
def test_download_warnsize_request_meta(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_warnsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
@coroutine_test
async def test_sitemap_urls(self):
class TestSpider(self.spider_class):
name = "test"
sitemap_urls = ["https://toscrape.com/sitemap.xml"]
crawler = get_crawler(TestSpider)
spider = TestSpider.from_crawler(crawler)
with warnings.catch_warnings():
warnings.simplefilter("error")
requests = [request async for request in spider.start()]
assert len(requests) == 1
request = requests[0]
assert request.url == "https://toscrape.com/sitemap.xml"
assert request.dont_filter is False
assert request.callback == spider._parse_sitemap
class TestDeprecation:
def test_crawl_spider(self):
assert issubclass(CrawlSpider, Spider)
assert isinstance(CrawlSpider(name="foo"), Spider)
class TestNoParseMethodSpider: class TestNoParseMethodSpider:
spider_class = Spider spider_class = Spider

307
tests/test_spider_crawl.py Normal file
View File

@ -0,0 +1,307 @@
from __future__ import annotations
import re
import warnings
from logging import ERROR
from testfixtures import LogCapture
from w3lib.url import safe_url_string
from scrapy.http import HtmlResponse, Request, TextResponse
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule, Spider
from scrapy.utils.test import get_crawler
from tests.test_spider import TestSpider
from tests.utils.decorators import inline_callbacks_test
class TestCrawlSpider(TestSpider):
test_body = b"""<html><head><title>Page title</title></head>
<body>
<p><a href="item/12.html">Item 12</a></p>
<div class='links'>
<p><a href="/about.html">About us</a></p>
</div>
<div>
<p><a href="/nofollow.html">This shouldn't be followed</a></p>
</div>
</body></html>"""
spider_class = CrawlSpider
def test_rule_without_link_extractor(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(),)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
def test_process_links(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
def dummy_process_links(self, links):
return links
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
def test_process_links_filter(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
_test_regex = re.compile("nofollow")
def filter_process_links(self, links):
return [link for link in links if not self._test_regex.search(link.url)]
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 2
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
]
def test_process_links_generator(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
def dummy_process_links(self, links):
yield from links
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
def test_process_request(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
def process_request_change_domain(request, response):
return request.replace(url=request.url.replace(".org", ".com"))
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (
Rule(LinkExtractor(), process_request=process_request_change_domain),
)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.com/somepage/item/12.html",
"http://example.com/about.html",
"http://example.com/nofollow.html",
]
def test_process_request_with_response(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
def process_request_meta_response_class(request, response):
request.meta["response_class"] = response.__class__.__name__
return request
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (
Rule(
LinkExtractor(), process_request=process_request_meta_response_class
),
)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
assert [r.meta["response_class"] for r in output] == [
"HtmlResponse",
"HtmlResponse",
"HtmlResponse",
]
def test_process_request_instance_method(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (Rule(LinkExtractor(), process_request="process_request_upper"),)
def process_request_upper(self, request, response):
return request.replace(url=request.url.upper())
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),
safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),
safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),
]
def test_process_request_instance_method_with_response(self):
response = HtmlResponse(
"http://example.org/somepage/index.html", body=self.test_body
)
class _CrawlSpider(self.spider_class):
name = "test"
allowed_domains = ["example.org"]
rules = (
Rule(
LinkExtractor(),
process_request="process_request_meta_response_class",
),
)
def process_request_meta_response_class(self, request, response):
request.meta["response_class"] = response.__class__.__name__
return request
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
assert len(output) == 3
assert all(isinstance(r, Request) for r in output)
assert [r.url for r in output] == [
"http://example.org/somepage/item/12.html",
"http://example.org/about.html",
"http://example.org/nofollow.html",
]
assert [r.meta["response_class"] for r in output] == [
"HtmlResponse",
"HtmlResponse",
"HtmlResponse",
]
def test_follow_links_attribute_population(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
assert hasattr(spider, "_follow_links")
assert spider._follow_links
settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False}
crawler = get_crawler(settings_dict=settings_dict)
spider = self.spider_class.from_crawler(crawler, "example.com")
assert hasattr(spider, "_follow_links")
assert not spider._follow_links
@inline_callbacks_test
def test_start_url(self):
class TestSpider(self.spider_class):
name = "test"
start_url = "https://www.example.com"
crawler = get_crawler(TestSpider)
with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log:
yield crawler.crawl()
assert "Error while reading start items and requests" in str(log)
assert "did you miss an 's'?" in str(log)
def test_parse_response_use(self):
class _CrawlSpider(CrawlSpider):
name = "test"
start_urls = "https://www.example.com"
_follow_links = False
with warnings.catch_warnings(record=True) as w:
spider = _CrawlSpider()
assert len(w) == 0
spider._parse_response(
TextResponse(spider.start_urls, body=b""), None, None
)
assert len(w) == 1
def test_parse_response_override(self):
class _CrawlSpider(CrawlSpider):
def _parse_response(self, response, callback, cb_kwargs, follow=True):
pass
name = "test"
start_urls = "https://www.example.com"
_follow_links = False
with warnings.catch_warnings(record=True) as w:
assert len(w) == 0
spider = _CrawlSpider()
assert len(w) == 1
spider._parse_response(
TextResponse(spider.start_urls, body=b""), None, None
)
assert len(w) == 1
def test_parse_with_rules(self):
class _CrawlSpider(CrawlSpider):
name = "test"
start_urls = "https://www.example.com"
with warnings.catch_warnings(record=True) as w:
spider = _CrawlSpider()
spider.parse_with_rules(
TextResponse(spider.start_urls, body=b""), None, None
)
assert len(w) == 0
class TestDeprecation:
def test_crawl_spider(self):
assert issubclass(CrawlSpider, Spider)
assert isinstance(CrawlSpider(name="foo"), Spider)

349
tests/test_spider_sitemap.py Normal file
View File

@ -0,0 +1,349 @@
from __future__ import annotations
import gzip
import warnings
from datetime import datetime
from io import BytesIO
from logging import WARNING
from pathlib import Path
import pytest
from testfixtures import LogCapture
from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse
from scrapy.spiders import SitemapSpider
from scrapy.utils.test import get_crawler
from tests import tests_datadir
from tests.test_spider import TestSpider
from tests.utils.decorators import coroutine_test
class TestSitemapSpider(TestSpider):
spider_class = SitemapSpider
BODY = b"SITEMAP"
f = BytesIO()
g = gzip.GzipFile(fileobj=f, mode="w+b")
g.write(BODY)
g.close()
GZBODY = f.getvalue()
def assertSitemapBody(self, response: Response, body: bytes | None) -> None:
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
assert spider._get_sitemap_body(response) == body
def test_get_sitemap_body(self):
r = XmlResponse(url="http://www.example.com/", body=self.BODY)
self.assertSitemapBody(r, self.BODY)
r = HtmlResponse(url="http://www.example.com/", body=self.BODY)
self.assertSitemapBody(r, None)
r = Response(url="http://www.example.com/favicon.ico", body=self.BODY)
self.assertSitemapBody(r, None)
def test_get_sitemap_body_gzip_headers(self):
r = Response(
url="http://www.example.com/sitemap",
body=self.GZBODY,
headers={"content-type": "application/gzip"},
request=Request("http://www.example.com/sitemap"),
)
self.assertSitemapBody(r, self.BODY)
def test_get_sitemap_body_xml_url(self):
r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY)
self.assertSitemapBody(r, self.BODY)
def test_get_sitemap_body_xml_url_compressed(self):
r = Response(
url="http://www.example.com/sitemap.xml.gz",
body=self.GZBODY,
request=Request("http://www.example.com/sitemap"),
)
self.assertSitemapBody(r, self.BODY)
# .xml.gz but body decoded by HttpCompression middleware already
r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY)
self.assertSitemapBody(r, self.BODY)
def test_get_sitemap_urls_from_robotstxt(self):
robots = b"""# Sitemap files
Sitemap: http://example.com/sitemap.xml
Sitemap: http://example.com/sitemap-product-index.xml
Sitemap: HTTP://example.com/sitemap-uppercase.xml
Sitemap: /sitemap-relative-url.xml
"""
r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://example.com/sitemap.xml",
"http://example.com/sitemap-product-index.xml",
"http://example.com/sitemap-uppercase.xml",
"http://www.example.com/sitemap-relative-url.xml",
]
def test_alternate_url_locs(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>http://www.example.com/english/</loc>
<xhtml:link rel="alternate" hreflang="de"
href="http://www.example.com/deutsch/"/>
<xhtml:link rel="alternate" hreflang="de-ch"
href="http://www.example.com/schweiz-deutsch/"/>
<xhtml:link rel="alternate" hreflang="it"
href="http://www.example.com/italiano/"/>
<xhtml:link rel="alternate" hreflang="it"/><!-- wrong tag without href -->
</url>
</urlset>"""
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/english/"
]
spider.sitemap_alternate_links = True
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/english/",
"http://www.example.com/deutsch/",
"http://www.example.com/schweiz-deutsch/",
"http://www.example.com/italiano/",
]
def test_sitemap_filter(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>http://www.example.com/english/</loc>
<lastmod>2010-01-01</lastmod>
</url>
<url>
<loc>http://www.example.com/portuguese/</loc>
<lastmod>2005-01-01</lastmod>
</url>
</urlset>"""
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
if date_time.year > 2008:
yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/english/",
"http://www.example.com/portuguese/",
]
spider = FilteredSitemapSpider("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/english/"
]
def test_sitemap_filter_with_alternate_links(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
xmlns:xhtml="http://www.w3.org/1999/xhtml">
<url>
<loc>http://www.example.com/english/article_1/</loc>
<lastmod>2010-01-01</lastmod>
<xhtml:link rel="alternate" hreflang="de"
href="http://www.example.com/deutsch/article_1/"/>
</url>
<url>
<loc>http://www.example.com/english/article_2/</loc>
<lastmod>2015-01-01</lastmod>
</url>
</urlset>"""
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
for entry in entries:
alternate_links = entry.get("alternate", ())
for link in alternate_links:
if "/deutsch/" in link:
entry["loc"] = link
yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/english/article_1/",
"http://www.example.com/english/article_2/",
]
spider = FilteredSitemapSpider("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/deutsch/article_1/"
]
def test_sitemapindex_filter(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap>
<loc>http://www.example.com/sitemap1.xml</loc>
<lastmod>2004-01-01T20:00:00+00:00</lastmod>
</sitemap>
<sitemap>
<loc>http://www.example.com/sitemap2.xml</loc>
<lastmod>2005-01-01</lastmod>
</sitemap>
</sitemapindex>"""
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
for entry in entries:
date_time = datetime.strptime(
entry["lastmod"].split("T")[0], "%Y-%m-%d"
)
if date_time.year > 2004:
yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/sitemap1.xml",
"http://www.example.com/sitemap2.xml",
]
spider = FilteredSitemapSpider("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://www.example.com/sitemap2.xml"
]
def test_compression_bomb_setting(self):
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
assert spider._get_sitemap_body(response) is None
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr(self):
class DownloadMaxSizeSpider(self.spider_class):
download_maxsize = 10_000_000
crawler = get_crawler()
spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
assert spider._get_sitemap_body(response) is None
def test_compression_bomb_request_meta(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_maxsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
assert spider._get_sitemap_body(response) is None
def test_download_warnsize_setting(self):
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(url="https://example.com")
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_download_warnsize_spider_attr(self):
class DownloadWarnSizeSpider(self.spider_class):
download_warnsize = 10_000_000
crawler = get_crawler()
spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_warnsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
def test_download_warnsize_request_meta(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, "example.com")
body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
body = body_path.read_bytes()
request = Request(
url="https://example.com", meta={"download_warnsize": 10_000_000}
)
response = Response(url="https://example.com", body=body, request=request)
with LogCapture(
"scrapy.spiders.sitemap", propagate=False, level=WARNING
) as log:
spider._get_sitemap_body(response)
log.check(
(
"scrapy.spiders.sitemap",
"WARNING",
(
"<200 https://example.com> body size after decompression "
"(11511612 B) is larger than the download warning size "
"(10000000 B)."
),
),
)
@coroutine_test
async def test_sitemap_urls(self):
class TestSpider(self.spider_class):
name = "test"
sitemap_urls = ["https://toscrape.com/sitemap.xml"]
crawler = get_crawler(TestSpider)
spider = TestSpider.from_crawler(crawler)
with warnings.catch_warnings():
warnings.simplefilter("error")
requests = [request async for request in spider.start()]
assert len(requests) == 1
request = requests[0]
assert request.url == "https://toscrape.com/sitemap.xml"
assert request.dont_filter is False
assert request.callback == spider._parse_sitemap