RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'scrapy/master' into integrate-mime

This commit is contained in:
Adrián Chaves 2024-02-08 17:55:02 +01:00
parent 2d9857b441 1c9d308acc
commit 8a3fd6bea5
32 changed files with 543 additions and 551 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/contributing.rst
View File

@ -178,7 +178,7 @@ Scrapy:
* We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
There is a hook in the pre-commit config
that will automatically format your code before every commit. You can also
run black manually with ``tox -e black``.
run black manually with ``tox -e pre-commit``.
* Don't put your name in the code you contribute; git provides enough
metadata to identify author of the code.

17
docs/faq.rst
View File

@ -405,6 +405,23 @@ or :class:`~scrapy.signals.headers_received` signals and raising a
:ref:`topics-stop-response-download` topic for additional information and examples.
.. _faq-blank-request:
How can I make a blank request?
-------------------------------
.. code-block:: python
from scrapy import Request
blank_request = Request("data:,")
In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
in-line in web pages as if they were external resources. The "data:" scheme with an empty
content (",") essentially creates a request to a data URL without any specific content.
Running ``runspider`` I get ``error: No spider found in file: <filename>``
--------------------------------------------------------------------------

6
docs/news.rst
View File

@ -62,6 +62,9 @@ Deprecation removals
1.0.0, use :attr:`CrawlerRunner.spider_loader
<scrapy.crawler.CrawlerRunner.spider_loader>` instead. (:issue:`6010`)
- The :func:`scrapy.utils.response.response_httprepr` function, deprecated in
Scrapy 2.6.0, has now been removed. (:issue:`6111`)
Deprecations
~~~~~~~~~~~~
@ -1157,6 +1160,9 @@ Deprecations
Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
first to set the :class:`~scrapy.Spider` object.
- :func:`scrapy.utils.response.response_httprepr` is now deprecated.
(:issue:`4972`)
New features
~~~~~~~~~~~~

8
docs/requirements.txt
View File

@ -1,4 +1,4 @@
sphinx==5.0.2
sphinx-hoverxref==1.1.1
sphinx-notfound-page==0.8
sphinx-rtd-theme==1.0.0
sphinx==6.2.1
sphinx-hoverxref==1.3.0
sphinx-notfound-page==1.0.0
sphinx-rtd-theme==2.0.0

54
docs/topics/request-response.rst
View File

@ -469,60 +469,6 @@ import path.
.. autoclass:: scrapy.utils.request.RequestFingerprinter
.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
REQUEST_FINGERPRINTER_IMPLEMENTATION
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. versionadded:: 2.7
Default: ``'2.6'``
Determines which request fingerprinting algorithm is used by the default
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
Possible values are:
- ``'2.6'`` (default)
This implementation uses the same request fingerprinting algorithm as
Scrapy 2.6 and earlier versions.
Even though this is the default value for backward compatibility reasons,
it is a deprecated value.
- ``'2.7'``
This implementation was introduced in Scrapy 2.7 to fix an issue of the
previous implementation.
New projects should use this value. The :command:`startproject` command
sets this value in the generated ``settings.py`` file.
If you are using the default value (``'2.6'``) for this setting, and you are
using Scrapy components where changing the request fingerprinting algorithm
would cause undesired results, you need to carefully decide when to change the
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
setting to a custom request fingerprinter class that implements the 2.6 request
fingerprinting algorithm and does not log this warning (
:ref:`2.6-request-fingerprinter` includes an example implementation of such a
class).
Scenarios where changing the request fingerprinting algorithm may cause
undesired results include, for example, using the HTTP cache middleware (see
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
Changing the request fingerprinting algorithm would invalidate the current
cache, requiring you to redownload all requests again.
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
your settings to switch already to the request fingerprinting implementation
that will be the only request fingerprinting implementation available in a
future version of Scrapy, and remove the deprecation warning triggered by using
the default value (``'2.6'``).
.. _2.6-request-fingerprinter:
.. _custom-request-fingerprinter:
Writing your own request fingerprinter

59
scrapy/commands/parse.py
View File

@ -1,3 +1,4 @@
import functools
import inspect
import json
import logging
@ -251,39 +252,40 @@ class Command(BaseRunSpiderCommand):
return scraped_data
def _get_callback(self, *, spider, opts, response=None):
cb = None
if response:
cb = response.meta["_callback"]
if not cb:
if opts.callback:
cb = opts.callback
elif response and opts.rules and self.first_response == response:
cb = self.get_callback_from_rules(spider, response)
if not cb:
raise ValueError(
f"Cannot find a rule that matches {response.url!r} in spider: "
f"{spider.name}"
)
else:
cb = "parse"
if not callable(cb):
cb_method = getattr(spider, cb, None)
if callable(cb_method):
cb = cb_method
else:
raise ValueError(
f"Cannot find callback {cb!r} in spider: {spider.name}"
)
return cb
def prepare_request(self, spider, request, opts):
def callback(response, **cb_kwargs):
# memorize first request
if not self.first_response:
self.first_response = response
# determine real callback
cb = response.meta["_callback"]
if not cb:
if opts.callback:
cb = opts.callback
elif opts.rules and self.first_response == response:
cb = self.get_callback_from_rules(spider, response)
if not cb:
logger.error(
"Cannot find a rule that matches %(url)r in spider: %(spider)s",
{"url": response.url, "spider": spider.name},
)
return
else:
cb = "parse"
if not callable(cb):
cb_method = getattr(spider, cb, None)
if callable(cb_method):
cb = cb_method
else:
logger.error(
"Cannot find callback %(callback)r in spider: %(spider)s",
{"callback": cb, "spider": spider.name},
)
return
cb = self._get_callback(spider=spider, opts=opts, response=response)
# parse items and requests
depth = response.meta["_depth"]
@ -303,6 +305,9 @@ class Command(BaseRunSpiderCommand):
request.meta["_depth"] = 1
request.meta["_callback"] = request.callback
if not request.callback and not opts.rules:
cb = self._get_callback(spider=spider, opts=opts)
functools.update_wrapper(callback, cb)
request.callback = callback
return request

3
scrapy/core/spidermw.py
View File

@ -103,8 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
) -> Union[Generator, AsyncGenerator]:
def process_sync(iterable: Iterable) -> Generator:
try:
for r in iterable:
yield r
yield from iterable
except Exception as ex:
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index

42
scrapy/crawler.py
View File

@ -178,6 +178,48 @@ class Crawler:
assert self.engine
yield maybeDeferred(self.engine.stop)
@staticmethod
def _get_component(component_class, components):
for component in components:
if isinstance(component, component_class):
return component
return None
def get_addon(self, cls):
return self._get_component(cls, self.addons.addons)
def get_downloader_middleware(self, cls):
if not self.engine:
raise RuntimeError(
"Crawler.get_downloader_middleware() can only be called after "
"the crawl engine has been created."
)
return self._get_component(cls, self.engine.downloader.middleware.middlewares)
def get_extension(self, cls):
if not self.extensions:
raise RuntimeError(
"Crawler.get_extension() can only be called after the "
"extension manager has been created."
)
return self._get_component(cls, self.extensions.middlewares)
def get_item_pipeline(self, cls):
if not self.engine:
raise RuntimeError(
"Crawler.get_item_pipeline() can only be called after the "
"crawl engine has been created."
)
return self._get_component(cls, self.engine.scraper.itemproc.middlewares)
def get_spider_middleware(self, cls):
if not self.engine:
raise RuntimeError(
"Crawler.get_spider_middleware() can only be called after the "
"crawl engine has been created."
)
return self._get_component(cls, self.engine.scraper.spidermw.middlewares)
class CrawlerRunner:
"""

6
scrapy/extensions/memusage.py
View File

@ -128,9 +128,9 @@ class MemoryUsage:
def _send_report(self, rcpts, subject):
"""send notification mail with some additional useful info"""
stats = self.crawler.stats
s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n"
s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n"
s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n"
s += (
"ENGINE STATUS ------------------------------------------------------- \r\n"

7
scrapy/pipelines/files.py
View File

@ -8,7 +8,6 @@ import functools
import hashlib
import logging
import mimetypes
import os
import time
from collections import defaultdict
from contextlib import suppress
@ -66,7 +65,7 @@ class FSFilesStore:
absolute_path = self._get_filesystem_path(path)
try:
last_modified = absolute_path.stat().st_mtime
except os.error:
except OSError:
return {}
with absolute_path.open("rb") as f:
@ -340,7 +339,9 @@ class FilesPipeline(MediaPipeline):
DEFAULT_FILES_URLS_FIELD = "file_urls"
DEFAULT_FILES_RESULT_FIELD = "files"
def __init__(self, store_uri, download_func=None, settings=None):
def __init__(
self, store_uri: Union[str, PathLike], download_func=None, settings=None
):
store_uri = _to_string(store_uri)
if not store_uri:
raise NotConfigured

7
scrapy/pipelines/images.py
View File

@ -8,7 +8,8 @@ import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
from typing import Dict, Tuple
from os import PathLike
from typing import Dict, Tuple, Union
from itemadapter import ItemAdapter
@ -53,7 +54,9 @@ class ImagesPipeline(FilesPipeline):
DEFAULT_IMAGES_URLS_FIELD = "image_urls"
DEFAULT_IMAGES_RESULT_FIELD = "images"
def __init__(self, store_uri, download_func=None, settings=None):
def __init__(
self, store_uri: Union[str, PathLike], download_func=None, settings=None
):
try:
from PIL import Image

2
scrapy/settings/default_settings.py
View File

@ -260,7 +260,7 @@ REFERER_ENABLED = True
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6"
REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests

3
scrapy/spiders/crawl.py
View File

@ -131,8 +131,7 @@ class CrawlSpider(Spider):
def _handle_failure(self, failure, errback):
if errback:
results = errback(failure) or ()
for request_or_item in iterate_spider_output(results):
yield request_or_item
yield from iterate_spider_output(results)
def _compile_rules(self):
self._rules = []

6
scrapy/spiders/feed.py
View File

@ -58,8 +58,7 @@ class XMLFeedSpider(Spider):
for selector in nodes:
ret = iterate_spider_output(self.parse_node(response, selector))
for result_item in self.process_results(response, ret):
yield result_item
yield from self.process_results(response, ret)
def _parse(self, response, **kwargs):
if not hasattr(self, "parse_node"):
@ -133,8 +132,7 @@ class CSVFeedSpider(Spider):
response, self.delimiter, self.headers, quotechar=self.quotechar
):
ret = iterate_spider_output(self.parse_row(response, row))
for result_item in self.process_results(response, ret):
yield result_item
yield from self.process_results(response, ret)
def _parse(self, response, **kwargs):
if not hasattr(self, "parse_row"):

3
scrapy/spiders/sitemap.py
View File

@ -33,8 +33,7 @@ class SitemapSpider(Spider):
attributes, for example, you can filter locs with lastmod greater
than a given date (see docs).
"""
for entry in entries:
yield entry
yield from entries
def _parse_sitemap(self, response):
if response.url.endswith("/robots.txt"):

1
scrapy/templates/project/module/settings.py.tmpl
View File

@ -88,6 +88,5 @@ ROBOTSTXT_OBEY = True
#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
# Set settings whose default value is deprecated to a future-proof value
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
FEED_EXPORT_ENCODING = "utf-8"

3
scrapy/utils/python.py
View File

@ -59,8 +59,7 @@ def iflatten(x: Iterable) -> Iterable:
Similar to ``.flatten()``, but returns iterator instead"""
for el in x:
if is_listlike(el):
for el_ in iflatten(el):
yield el_
yield from iflatten(el)
else:
yield el

150
scrapy/utils/request.py
View File

@ -34,9 +34,6 @@ from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
from scrapy.crawler import Crawler
_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
_deprecated_fingerprint_cache = WeakKeyDictionary()
def _serialize_headers(
headers: Iterable[bytes], request: Request
@ -44,122 +41,7 @@ def _serialize_headers(
for header in headers:
if header in request.headers:
yield header
for value in request.headers.getlist(header):
yield value
def request_fingerprint(
request: Request,
include_headers: Optional[Iterable[Union[bytes, str]]] = None,
keep_fragments: bool = False,
) -> str:
"""
Return the request fingerprint as an hexadecimal string.
The request fingerprint is a hash that uniquely identifies the resource the
request points to. For example, take the following two urls:
http://www.example.com/query?id=111&cat=222
http://www.example.com/query?cat=222&id=111
Even though those are two different URLs both point to the same resource
and are equivalent (i.e. they should return the same response).
Another example are cookies used to store session ids. Suppose the
following page is only accessible to authenticated users:
http://www.example.com/members/offers.html
Lots of sites use a cookie to store the session id, which adds a random
component to the HTTP Request and thus should be ignored when calculating
the fingerprint.
For this reason, request headers are ignored by default when calculating
the fingerprint. If you want to include specific headers use the
include_headers argument, which is a list of Request headers to include.
Also, servers usually ignore fragments in urls when handling requests,
so they are also ignored by default when calculating the fingerprint.
If you want to include them, set the keep_fragments argument to True
(for instance when handling requests with a headless browser).
"""
if include_headers or keep_fragments:
message = (
"Call to deprecated function "
"scrapy.utils.request.request_fingerprint().\n"
"\n"
"If you are using this function in a Scrapy component because you "
"need a non-default fingerprinting algorithm, and you are OK "
"with that non-default fingerprinting algorithm being used by "
"all Scrapy components and not just the one calling this "
"function, use crawler.request_fingerprinter.fingerprint() "
"instead in your Scrapy component (you can get the crawler "
"object from the 'from_crawler' class method), and use the "
"'REQUEST_FINGERPRINTER_CLASS' setting to configure your "
"non-default fingerprinting algorithm.\n"
"\n"
"Otherwise, consider using the "
"scrapy.utils.request.fingerprint() function instead.\n"
"\n"
"If you switch to 'fingerprint()', or assign the "
"'REQUEST_FINGERPRINTER_CLASS' setting a class that uses "
"'fingerprint()', the generated fingerprints will not only be "
"bytes instead of a string, but they will also be different from "
"those generated by 'request_fingerprint()'. Before you switch, "
"make sure that you understand the consequences of this (e.g. "
"cache invalidation) and are OK with them; otherwise, consider "
"implementing your own function which returns the same "
"fingerprints as the deprecated 'request_fingerprint()' function."
)
else:
message = (
"Call to deprecated function "
"scrapy.utils.request.request_fingerprint().\n"
"\n"
"If you are using this function in a Scrapy component, and you "
"are OK with users of your component changing the fingerprinting "
"algorithm through settings, use "
"crawler.request_fingerprinter.fingerprint() instead in your "
"Scrapy component (you can get the crawler object from the "
"'from_crawler' class method).\n"
"\n"
"Otherwise, consider using the "
"scrapy.utils.request.fingerprint() function instead.\n"
"\n"
"Either way, the resulting fingerprints will be returned as "
"bytes, not as a string, and they will also be different from "
"those generated by 'request_fingerprint()'. Before you switch, "
"make sure that you understand the consequences of this (e.g. "
"cache invalidation) and are OK with them; otherwise, consider "
"implementing your own function which returns the same "
"fingerprints as the deprecated 'request_fingerprint()' function."
)
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
processed_include_headers: Optional[Tuple[bytes, ...]] = None
if include_headers:
processed_include_headers = tuple(
to_bytes(h.lower()) for h in sorted(include_headers)
)
cache = _deprecated_fingerprint_cache.setdefault(request, {})
cache_key = (processed_include_headers, keep_fragments)
if cache_key not in cache:
fp = hashlib.sha1()
fp.update(to_bytes(request.method))
fp.update(
to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
)
fp.update(request.body or b"")
if processed_include_headers:
for part in _serialize_headers(processed_include_headers, request):
fp.update(part)
cache[cache_key] = fp.hexdigest()
return cache[cache_key]
def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
return bytes.fromhex(request_fingerprint(*args, **kwargs))
yield from request.headers.getlist(header)
_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
@ -259,33 +141,15 @@ class RequestFingerprinter:
"REQUEST_FINGERPRINTER_IMPLEMENTATION"
)
else:
implementation = "2.6"
if implementation == "2.6":
implementation = "SENTINEL"
if implementation != "SENTINEL":
message = (
"'2.6' is a deprecated value for the "
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n"
"\n"
"It is also the default value. In other words, it is normal "
"to get this warning if you have not defined a value for the "
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so "
"for backward compatibility reasons, but it will change in a "
"future version of Scrapy.\n"
"\n"
"See the documentation of the "
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for "
"information on how to handle this deprecation."
"'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
"And it will be removed in future version of Scrapy."
)
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
self._fingerprint = _request_fingerprint_as_bytes
elif implementation == "2.7":
self._fingerprint = fingerprint
else:
raise ValueError(
f"Got an invalid value on setting "
f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': "
f"{implementation!r}. Valid values are '2.6' (deprecated) "
f"and '2.7'."
)
self._fingerprint = fingerprint
def fingerprint(self, request: Request) -> bytes:
return self._fingerprint(request)

2
scrapy/utils/test.py
View File

@ -86,8 +86,6 @@ def get_crawler(
# Set by default settings that prevent deprecation warnings.
settings: Dict[str, Any] = {}
if prevent_warnings:
settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7"
settings.update(settings_dict or {})
runner = CrawlerRunner(settings)
crawler = runner.create_crawler(spidercls or TestSpider)

3
tests/spiders.py
View File

@ -301,8 +301,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
def parse(self, response):
self.seedsseen.append(response.meta.get("seed"))
for req in super().parse(response):
yield req
yield from super().parse(response)
class SingleRequestSpider(MetaSpider):

4
tests/test_command_check.py
View File

@ -16,11 +16,11 @@ import scrapy
class CheckSpider(scrapy.Spider):
name = '{self.spider_name}'
start_urls = ['http://toscrape.com']
start_urls = ['data:,']
def parse(self, response, **cb_kwargs):
\"\"\"
@url http://toscrape.com
@url data:,
{contracts}
\"\"\"
{parse_def}

18
tests/test_command_parse.py
View File

@ -78,9 +78,21 @@ class AsyncDefAsyncioGenExcSpider(scrapy.Spider):
if i > 5:
raise ValueError("Stopping the processing")
class CallbackSignatureDownloaderMiddleware:
def process_request(self, request, spider):
from inspect import signature
spider.logger.debug(f"request.callback signature: {{signature(request.callback)}}")
class MySpider(scrapy.Spider):
name = '{self.spider_name}'
custom_settings = {{
"DOWNLOADER_MIDDLEWARES": {{
CallbackSignatureDownloaderMiddleware: 0,
}}
}}
def parse(self, response):
if getattr(self, 'test_arg', None):
self.logger.debug('It Works!')
@ -220,7 +232,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
self.url("/html"),
]
)
self.assertIn("DEBUG: It Works!", _textmode(stderr))
log = _textmode(stderr)
self.assertIn("DEBUG: It Works!", log)
self.assertIn(
"DEBUG: request.callback signature: (response, foo=None, key=None)", log
)
@defer.inlineCallbacks
def test_request_without_meta(self):

388
tests/test_crawler.py
View File

@ -12,12 +12,13 @@ import pytest
from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from pytest import mark, raises
from twisted.internet import defer
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.trial import unittest
from w3lib import __version__ as w3lib_version
from zope.interface.exceptions import MultipleInvalid
import scrapy
from scrapy import Spider
from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions import telnet
@ -29,6 +30,19 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer, get_mockserver_env
# To prevent warnings.
BASE_SETTINGS = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
def get_raw_crawler(spidercls=None, settings_dict=None):
"""get_crawler alternative that only calls the __init__ method of the
crawler."""
settings = Settings()
settings.setdict(settings_dict or {})
return Crawler(spidercls or DefaultSpider, settings)
class BaseCrawlerTest(unittest.TestCase):
def assertOptionIsDefault(self, settings, key):
@ -39,7 +53,7 @@ class BaseCrawlerTest(unittest.TestCase):
class CrawlerTestCase(BaseCrawlerTest):
def test_populate_spidercls_settings(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
project_settings = {"TEST1": "project", "TEST3": "project"}
project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"}
class CustomSettingsSpider(DefaultSpider):
custom_settings = spider_settings
@ -71,9 +85,9 @@ class CrawlerTestCase(BaseCrawlerTest):
with raises(ValueError):
Crawler(DefaultSpider())
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler_crawl_twice_deprecated(self):
crawler = Crawler(NoRequestsSpider)
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
yield crawler.crawl()
with pytest.warns(
ScrapyDeprecationWarning,
@ -81,6 +95,358 @@ class CrawlerTestCase(BaseCrawlerTest):
):
yield crawler.crawl()
def test_get_addon(self):
class ParentAddon:
pass
class TrackingAddon(ParentAddon):
instances = []
def __init__(self):
TrackingAddon.instances.append(self)
def update_settings(self, settings):
pass
settings = {
**BASE_SETTINGS,
"ADDONS": {
TrackingAddon: 0,
},
}
crawler = get_crawler(settings_dict=settings)
self.assertEqual(len(TrackingAddon.instances), 1)
expected = TrackingAddon.instances[-1]
addon = crawler.get_addon(TrackingAddon)
self.assertEqual(addon, expected)
addon = crawler.get_addon(DefaultSpider)
self.assertIsNone(addon)
addon = crawler.get_addon(ParentAddon)
self.assertEqual(addon, expected)
class ChildAddon(TrackingAddon):
pass
addon = crawler.get_addon(ChildAddon)
self.assertIsNone(addon)
@inlineCallbacks
def test_get_downloader_middleware(self):
class ParentDownloaderMiddleware:
pass
class TrackingDownloaderMiddleware(ParentDownloaderMiddleware):
instances = []
def __init__(self):
TrackingDownloaderMiddleware.instances.append(self)
class MySpider(Spider):
name = "myspider"
@classmethod
def from_crawler(cls, crawler):
return cls(crawler=crawler)
def __init__(self, crawler):
self.crawler = crawler
def start_requests(self):
MySpider.result = crawler.get_downloader_middleware(MySpider.cls)
return
yield
settings = {
**BASE_SETTINGS,
"DOWNLOADER_MIDDLEWARES": {
TrackingDownloaderMiddleware: 0,
},
}
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = TrackingDownloaderMiddleware
yield crawler.crawl()
self.assertEqual(len(TrackingDownloaderMiddleware.instances), 1)
self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1])
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = DefaultSpider
yield crawler.crawl()
self.assertIsNone(MySpider.result)
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ParentDownloaderMiddleware
yield crawler.crawl()
self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1])
class ChildDownloaderMiddleware(TrackingDownloaderMiddleware):
pass
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ChildDownloaderMiddleware
yield crawler.crawl()
self.assertIsNone(MySpider.result)
def test_get_downloader_middleware_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
self.assertRaises(
RuntimeError, crawler.get_downloader_middleware, DefaultSpider
)
@inlineCallbacks
def test_get_downloader_middleware_no_engine(self):
class MySpider(Spider):
name = "myspider"
@classmethod
def from_crawler(cls, crawler):
try:
crawler.get_downloader_middleware(DefaultSpider)
except Exception as e:
MySpider.result = e
raise
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with raises(RuntimeError):
yield crawler.crawl()
@inlineCallbacks
def test_get_extension(self):
class ParentExtension:
pass
class TrackingExtension(ParentExtension):
instances = []
def __init__(self):
TrackingExtension.instances.append(self)
class MySpider(Spider):
name = "myspider"
@classmethod
def from_crawler(cls, crawler):
return cls(crawler=crawler)
def __init__(self, crawler):
self.crawler = crawler
def start_requests(self):
MySpider.result = crawler.get_extension(MySpider.cls)
return
yield
settings = {
**BASE_SETTINGS,
"EXTENSIONS": {
TrackingExtension: 0,
},
}
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = TrackingExtension
yield crawler.crawl()
self.assertEqual(len(TrackingExtension.instances), 1)
self.assertEqual(MySpider.result, TrackingExtension.instances[-1])
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = DefaultSpider
yield crawler.crawl()
self.assertIsNone(MySpider.result)
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ParentExtension
yield crawler.crawl()
self.assertEqual(MySpider.result, TrackingExtension.instances[-1])
class ChildExtension(TrackingExtension):
pass
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ChildExtension
yield crawler.crawl()
self.assertIsNone(MySpider.result)
def test_get_extension_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider)
@inlineCallbacks
def test_get_extension_no_engine(self):
class MySpider(Spider):
name = "myspider"
@classmethod
def from_crawler(cls, crawler):
try:
crawler.get_extension(DefaultSpider)
except Exception as e:
MySpider.result = e
raise
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with raises(RuntimeError):
yield crawler.crawl()
@inlineCallbacks
def test_get_item_pipeline(self):
class ParentItemPipeline:
pass
class TrackingItemPipeline(ParentItemPipeline):
instances = []
def __init__(self):
TrackingItemPipeline.instances.append(self)
class MySpider(Spider):
name = "myspider"
@classmethod
def from_crawler(cls, crawler):
return cls(crawler=crawler)
def __init__(self, crawler):
self.crawler = crawler
def start_requests(self):
MySpider.result = crawler.get_item_pipeline(MySpider.cls)
return
yield
settings = {
**BASE_SETTINGS,
"ITEM_PIPELINES": {
TrackingItemPipeline: 0,
},
}
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = TrackingItemPipeline
yield crawler.crawl()
self.assertEqual(len(TrackingItemPipeline.instances), 1)
self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1])
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = DefaultSpider
yield crawler.crawl()
self.assertIsNone(MySpider.result)
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ParentItemPipeline
yield crawler.crawl()
self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1])
class ChildItemPipeline(TrackingItemPipeline):
pass
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ChildItemPipeline
yield crawler.crawl()
self.assertIsNone(MySpider.result)
def test_get_item_pipeline_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider)
@inlineCallbacks
def test_get_item_pipeline_no_engine(self):
class MySpider(Spider):
name = "myspider"
@classmethod
def from_crawler(cls, crawler):
try:
crawler.get_item_pipeline(DefaultSpider)
except Exception as e:
MySpider.result = e
raise
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with raises(RuntimeError):
yield crawler.crawl()
@inlineCallbacks
def test_get_spider_middleware(self):
class ParentSpiderMiddleware:
pass
class TrackingSpiderMiddleware(ParentSpiderMiddleware):
instances = []
def __init__(self):
TrackingSpiderMiddleware.instances.append(self)
class MySpider(Spider):
name = "myspider"
@classmethod
def from_crawler(cls, crawler):
return cls(crawler=crawler)
def __init__(self, crawler):
self.crawler = crawler
def start_requests(self):
MySpider.result = crawler.get_spider_middleware(MySpider.cls)
return
yield
settings = {
**BASE_SETTINGS,
"SPIDER_MIDDLEWARES": {
TrackingSpiderMiddleware: 0,
},
}
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = TrackingSpiderMiddleware
yield crawler.crawl()
self.assertEqual(len(TrackingSpiderMiddleware.instances), 1)
self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1])
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = DefaultSpider
yield crawler.crawl()
self.assertIsNone(MySpider.result)
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ParentSpiderMiddleware
yield crawler.crawl()
self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1])
class ChildSpiderMiddleware(TrackingSpiderMiddleware):
pass
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ChildSpiderMiddleware
yield crawler.crawl()
self.assertIsNone(MySpider.result)
def test_get_spider_middleware_not_crawling(self):
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider)
@inlineCallbacks
def test_get_spider_middleware_no_engine(self):
class MySpider(Spider):
name = "myspider"
@classmethod
def from_crawler(cls, crawler):
try:
crawler.get_spider_middleware(DefaultSpider)
except Exception as e:
MySpider.result = e
raise
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with raises(RuntimeError):
yield crawler.crawl()
class SpiderSettingsTestCase(unittest.TestCase):
def test_spider_custom_settings(self):
@ -223,20 +589,20 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
def _runner(self):
return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"})
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler_runner_bootstrap_successful(self):
runner = self._runner()
yield runner.crawl(NoRequestsSpider)
self.assertFalse(runner.bootstrap_failed)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler_runner_bootstrap_successful_for_several(self):
runner = self._runner()
yield runner.crawl(NoRequestsSpider)
yield runner.crawl(NoRequestsSpider)
self.assertFalse(runner.bootstrap_failed)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler_runner_bootstrap_failed(self):
runner = self._runner()
@ -249,7 +615,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
self.assertTrue(runner.bootstrap_failed)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler_runner_bootstrap_failed_for_several(self):
runner = self._runner()
@ -264,7 +630,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
self.assertTrue(runner.bootstrap_failed)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler_runner_asyncio_enabled_true(self):
if self.reactor_pytest == "asyncio":
CrawlerRunner(
@ -531,7 +897,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
p.expect_exact("Spider closed (shutdown)")
p.wait()
@defer.inlineCallbacks
@inlineCallbacks
def test_shutdown_forced(self):
from twisted.internet import reactor
@ -543,7 +909,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
p.kill(sig)
p.expect_exact("shutting down gracefully")
# sending the second signal too fast often causes problems
d = defer.Deferred()
d = Deferred()
reactor.callLater(0.1, d.callback, None)
yield d
p.kill(sig)

2
tests/test_extension_periodic_log.py
View File

@ -67,7 +67,7 @@ def extension(settings=None):
class TestPeriodicLog(unittest.TestCase):
def test_extension_enabled(self):
# Expected that settings for this extension loaded succesfully
# Expected that settings for this extension loaded successfully
# And on certain conditions - extension raising NotConfigured
# "PERIODIC_LOG_STATS": True -> set to {"enabled": True}

8
tests/test_feedexport.py
View File

@ -673,8 +673,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
name = "testspider"
def parse(self, response):
for item in items:
yield item
yield from items
data = yield self.run_and_export(TestSpider, settings)
return data
@ -2300,7 +2299,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
content[feed["format"]].append(file.read_bytes())
finally:
self.tearDown()
defer.returnValue(content)
return content
@defer.inlineCallbacks
def assertExportedJsonLines(self, items, rows, settings=None):
@ -2696,8 +2695,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
name = "testspider"
def parse(self, response):
for item in items:
yield item
yield from items
with MockServer() as server:
TestSpider.start_urls = [server.url("/")]

3
tests/test_proxy_connect.py
View File

@ -31,6 +31,7 @@ sys.exit(mitmdump())
self.proc = Popen(
[
sys.executable,
"-u",
"-c",
script,
"--listen-host",
@ -46,7 +47,7 @@ sys.exit(mitmdump())
stdout=PIPE,
)
line = self.proc.stdout.readline().decode("utf-8")
host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)
host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1)
address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
return address

3
tests/test_spider.py
View File

@ -317,8 +317,7 @@ class CrawlSpiderTest(SpiderTest):
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
def dummy_process_links(self, links):
for link in links:
yield link
yield from links
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))

9
tests/test_spidermiddleware.py
View File

@ -170,8 +170,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
class ProcessSpiderOutputSimpleMiddleware:
def process_spider_output(self, response, result, spider):
for r in result:
yield r
yield from result
class ProcessSpiderOutputAsyncGenMiddleware:
@ -182,8 +181,7 @@ class ProcessSpiderOutputAsyncGenMiddleware:
class ProcessSpiderOutputUniversalMiddleware:
def process_spider_output(self, response, result, spider):
for r in result:
yield r
yield from result
async def process_spider_output_async(self, response, result, spider):
async for r in result:
@ -324,8 +322,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
class ProcessStartRequestsSimpleMiddleware:
def process_start_requests(self, start_requests, spider):
for r in start_requests:
yield r
yield from start_requests
class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):

3
tests/test_utils_defer.py
View File

@ -107,8 +107,7 @@ class DeferUtilsTest(unittest.TestCase):
class IterErrbackTest(unittest.TestCase):
def test_iter_errback_good(self):
def itergood():
for x in range(10):
yield x
yield from range(10)
errors = []
out = list(iter_errback(itergood(), errors.append))

2
tests/test_utils_iterators.py
View File

@ -524,6 +524,6 @@ class TestHelper(unittest.TestCase):
def _assert_type_and_value(self, a, b, obj):
self.assertTrue(
type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}"
type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}"
)
self.assertEqual(a, b)

262
tests/test_utils_request.py
View File

@ -2,22 +2,15 @@ import json
import unittest
import warnings
from hashlib import sha1
from typing import Dict, Mapping, Optional, Tuple, Union
from typing import Dict, Optional, Tuple, Union
from weakref import WeakKeyDictionary
import pytest
from w3lib.url import canonicalize_url
from scrapy.http import Request
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.python import to_bytes
from scrapy.utils.request import (
_deprecated_fingerprint_cache,
_fingerprint_cache,
_request_fingerprint_as_bytes,
fingerprint,
request_authenticate,
request_fingerprint,
request_httprepr,
request_to_curl,
)
@ -233,168 +226,6 @@ class FingerprintTest(unittest.TestCase):
self.assertEqual(actual, expected)
class RequestFingerprintTest(FingerprintTest):
function = staticmethod(request_fingerprint)
cache = _deprecated_fingerprint_cache
known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
(
Request("http://example.org"),
"b2e5245ef826fd9576c93bd6e392fce3133fab62",
{},
),
(
Request("https://example.org"),
"bd10a0a89ea32cdee77917320f1309b0da87e892",
{},
),
(
Request("https://example.org?a"),
"2fb7d48ae02f04b749f40caa969c0bc3c43204ce",
{},
),
(
Request("https://example.org?a=b"),
"42e5fe149b147476e3f67ad0670c57b4cc57856a",
{},
),
(
Request("https://example.org?a=b&a"),
"d23a9787cb56c6375c2cae4453c5a8c634526942",
{},
),
(
Request("https://example.org?a=b&a=c"),
"9a18a7a8552a9182b7f1e05d33876409e421e5c5",
{},
),
(
Request("https://example.org", method="POST"),
"ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6",
{},
),
(
Request("https://example.org", body=b"a"),
"4bb136e54e715a4ea7a9dd1101831765d33f2d60",
{},
),
(
Request("https://example.org", method="POST", body=b"a"),
"6c6595374a304b293be762f7b7be3f54e9947c65",
{},
),
(
Request("https://example.org#a", headers={"A": b"B"}),
"bd10a0a89ea32cdee77917320f1309b0da87e892",
{},
),
(
Request("https://example.org#a", headers={"A": b"B"}),
"515b633cb3ca502a33a9d8c890e889ec1e425e65",
{"include_headers": ["A"]},
),
(
Request("https://example.org#a", headers={"A": b"B"}),
"505c96e7da675920dfef58725e8c957dfdb38f47",
{"keep_fragments": True},
),
(
Request("https://example.org#a", headers={"A": b"B"}),
"d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da",
{"include_headers": ["A"], "keep_fragments": True},
),
(
Request("https://example.org/ab"),
"4e2870fee58582d6f81755e9b8fdefe3cba0c951",
{},
),
(
Request("https://example.org/a", body=b"b"),
"4e2870fee58582d6f81755e9b8fdefe3cba0c951",
{},
),
)
def setUp(self) -> None:
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
def tearDown(self) -> None:
warnings.simplefilter("default", ScrapyDeprecationWarning)
@pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
def test_part_separation(self):
super().test_part_separation()
class RequestFingerprintDeprecationTest(unittest.TestCase):
def test_deprecation_default_parameters(self):
with pytest.warns(ScrapyDeprecationWarning) as warnings:
request_fingerprint(Request("http://www.example.com"))
messages = [str(warning.message) for warning in warnings]
self.assertTrue(
any("Call to deprecated function" in message for message in messages)
)
self.assertFalse(any("non-default" in message for message in messages))
def test_deprecation_non_default_parameters(self):
with pytest.warns(ScrapyDeprecationWarning) as warnings:
request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
messages = [str(warning.message) for warning in warnings]
self.assertTrue(
any("Call to deprecated function" in message for message in messages)
)
self.assertTrue(any("non-default" in message for message in messages))
class RequestFingerprintAsBytesTest(FingerprintTest):
function = staticmethod(_request_fingerprint_as_bytes)
cache = _deprecated_fingerprint_cache
known_hashes = RequestFingerprintTest.known_hashes
def test_caching(self):
r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
self.assertEqual(
self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key])
)
@pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
def test_part_separation(self):
super().test_part_separation()
def test_hashes(self):
actual = [
self.function(request, **kwargs) for request, _, kwargs in self.known_hashes
]
expected = [
bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes
]
self.assertEqual(actual, expected)
_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
if include_headers:
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
cache = _fingerprint_cache_2_6.setdefault(request, {})
cache_key = (include_headers, keep_fragments)
if cache_key not in cache:
fp = sha1()
fp.update(to_bytes(request.method))
fp.update(
to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
)
fp.update(request.body or b"")
if include_headers:
for hdr in include_headers:
if hdr in request.headers:
fp.update(hdr)
for v in request.headers.getlist(hdr):
fp.update(v)
cache[cache_key] = fp.hexdigest()
return cache[cache_key]
REQUEST_OBJECTS_TO_TEST = (
Request("http://www.example.com/"),
Request("http://www.example.com/query?id=111&cat=222"),
@ -424,94 +255,16 @@ REQUEST_OBJECTS_TO_TEST = (
)
class BackwardCompatibilityTestCase(unittest.TestCase):
def test_function_backward_compatibility(self):
include_headers_to_test = (
None,
["Accept-Language"],
["accept-language", "sessionid"],
["SESSIONID", "Accept-Language"],
)
for request_object in REQUEST_OBJECTS_TO_TEST:
for include_headers in include_headers_to_test:
for keep_fragments in (False, True):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
fp = request_fingerprint(
request_object,
include_headers=include_headers,
keep_fragments=keep_fragments,
)
old_fp = request_fingerprint_2_6(
request_object,
include_headers=include_headers,
keep_fragments=keep_fragments,
)
self.assertEqual(fp, old_fp)
def test_component_backward_compatibility(self):
for request_object in REQUEST_OBJECTS_TO_TEST:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
crawler = get_crawler(prevent_warnings=False)
fp = crawler.request_fingerprinter.fingerprint(request_object)
old_fp = request_fingerprint_2_6(request_object)
self.assertEqual(fp.hex(), old_fp)
def test_custom_component_backward_compatibility(self):
"""Tests that the backward-compatible request fingerprinting class featured
in the documentation is indeed backward compatible and does not cause a
warning to be logged."""
class RequestFingerprinter:
cache = WeakKeyDictionary()
def fingerprint(self, request):
if request not in self.cache:
fp = sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url)))
fp.update(request.body or b"")
self.cache[request] = fp.digest()
return self.cache[request]
for request_object in REQUEST_OBJECTS_TO_TEST:
with warnings.catch_warnings() as logged_warnings:
settings = {
"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
}
crawler = get_crawler(settings_dict=settings)
fp = crawler.request_fingerprinter.fingerprint(request_object)
old_fp = request_fingerprint_2_6(request_object)
self.assertEqual(fp.hex(), old_fp)
self.assertFalse(logged_warnings)
class RequestFingerprinterTestCase(unittest.TestCase):
def test_default_implementation(self):
with warnings.catch_warnings(record=True) as logged_warnings:
crawler = get_crawler(prevent_warnings=False)
crawler = get_crawler()
request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
_request_fingerprint_as_bytes(request),
fingerprint(request),
)
self.assertTrue(logged_warnings)
def test_deprecated_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6",
}
with warnings.catch_warnings(record=True) as logged_warnings:
crawler = get_crawler(settings_dict=settings)
request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
_request_fingerprint_as_bytes(request),
)
self.assertTrue(logged_warnings)
def test_recommended_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
@ -522,14 +275,7 @@ class RequestFingerprinterTestCase(unittest.TestCase):
crawler.request_fingerprinter.fingerprint(request),
fingerprint(request),
)
self.assertFalse(logged_warnings)
def test_unknown_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",
}
with self.assertRaises(ValueError):
get_crawler(settings_dict=settings)
self.assertTrue(logged_warnings)
class CustomRequestFingerprinterTestCase(unittest.TestCase):

8
tox.ini
View File

@ -11,11 +11,7 @@ minversion = 1.7.0
deps =
-rtests/requirements.txt
# mitmproxy does not support PyPy
# Python 3.9+ requires mitmproxy >= 5.3.0
# mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
#mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
# The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
mitmproxy; implementation_name != 'pypy'
passenv =
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID
@ -88,7 +84,7 @@ deps =
xtractmime==0.2.0
-rtests/requirements.txt
# mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
# mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
# above, hence we do not install it in pinned environments at the moment
setenv =
_SCRAPY_PINNED=true