diff --git a/docs/contributing.rst b/docs/contributing.rst
index 2b3249601..d728338da 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -178,7 +178,7 @@ Scrapy:
* We use `black `_ for code formatting.
There is a hook in the pre-commit config
that will automatically format your code before every commit. You can also
- run black manually with ``tox -e black``.
+ run black manually with ``tox -e pre-commit``.
* Don't put your name in the code you contribute; git provides enough
metadata to identify author of the code.
diff --git a/docs/faq.rst b/docs/faq.rst
index 20dd814df..2113b0964 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -405,6 +405,23 @@ or :class:`~scrapy.signals.headers_received` signals and raising a
:ref:`topics-stop-response-download` topic for additional information and examples.
+.. _faq-blank-request:
+
+How can I make a blank request?
+-------------------------------
+
+.. code-block:: python
+
+ from scrapy import Request
+
+
+ blank_request = Request("data:,")
+
+In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
+in-line in web pages as if they were external resources. The "data:" scheme with an empty
+content (",") essentially creates a request to a data URL without any specific content.
+
+
Running ``runspider`` I get ``error: No spider found in file: ``
--------------------------------------------------------------------------
diff --git a/docs/news.rst b/docs/news.rst
index 65d9c5181..d90e32560 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -62,6 +62,9 @@ Deprecation removals
1.0.0, use :attr:`CrawlerRunner.spider_loader
` instead. (:issue:`6010`)
+- The :func:`scrapy.utils.response.response_httprepr` function, deprecated in
+ Scrapy 2.6.0, has now been removed. (:issue:`6111`)
+
Deprecations
~~~~~~~~~~~~
@@ -1157,6 +1160,9 @@ Deprecations
Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`
first to set the :class:`~scrapy.Spider` object.
+- :func:`scrapy.utils.response.response_httprepr` is now deprecated.
+ (:issue:`4972`)
+
New features
~~~~~~~~~~~~
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 9f9aef711..5f683d34c 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,4 +1,4 @@
-sphinx==5.0.2
-sphinx-hoverxref==1.1.1
-sphinx-notfound-page==0.8
-sphinx-rtd-theme==1.0.0
+sphinx==6.2.1
+sphinx-hoverxref==1.3.0
+sphinx-notfound-page==1.0.0
+sphinx-rtd-theme==2.0.0
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index a75da1d57..4440783cf 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -469,60 +469,6 @@ import path.
.. autoclass:: scrapy.utils.request.RequestFingerprinter
-
-.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
-
-REQUEST_FINGERPRINTER_IMPLEMENTATION
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. versionadded:: 2.7
-
-Default: ``'2.6'``
-
-Determines which request fingerprinting algorithm is used by the default
-request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
-
-Possible values are:
-
-- ``'2.6'`` (default)
-
- This implementation uses the same request fingerprinting algorithm as
- Scrapy 2.6 and earlier versions.
-
- Even though this is the default value for backward compatibility reasons,
- it is a deprecated value.
-
-- ``'2.7'``
-
- This implementation was introduced in Scrapy 2.7 to fix an issue of the
- previous implementation.
-
- New projects should use this value. The :command:`startproject` command
- sets this value in the generated ``settings.py`` file.
-
-If you are using the default value (``'2.6'``) for this setting, and you are
-using Scrapy components where changing the request fingerprinting algorithm
-would cause undesired results, you need to carefully decide when to change the
-value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
-setting to a custom request fingerprinter class that implements the 2.6 request
-fingerprinting algorithm and does not log this warning (
-:ref:`2.6-request-fingerprinter` includes an example implementation of such a
-class).
-
-Scenarios where changing the request fingerprinting algorithm may cause
-undesired results include, for example, using the HTTP cache middleware (see
-:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
-Changing the request fingerprinting algorithm would invalidate the current
-cache, requiring you to redownload all requests again.
-
-Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
-your settings to switch already to the request fingerprinting implementation
-that will be the only request fingerprinting implementation available in a
-future version of Scrapy, and remove the deprecation warning triggered by using
-the default value (``'2.6'``).
-
-
-.. _2.6-request-fingerprinter:
.. _custom-request-fingerprinter:
Writing your own request fingerprinter
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index ac937e464..c9f8586d3 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,3 +1,4 @@
+import functools
import inspect
import json
import logging
@@ -251,39 +252,40 @@ class Command(BaseRunSpiderCommand):
return scraped_data
+ def _get_callback(self, *, spider, opts, response=None):
+ cb = None
+ if response:
+ cb = response.meta["_callback"]
+ if not cb:
+ if opts.callback:
+ cb = opts.callback
+ elif response and opts.rules and self.first_response == response:
+ cb = self.get_callback_from_rules(spider, response)
+ if not cb:
+ raise ValueError(
+ f"Cannot find a rule that matches {response.url!r} in spider: "
+ f"{spider.name}"
+ )
+ else:
+ cb = "parse"
+
+ if not callable(cb):
+ cb_method = getattr(spider, cb, None)
+ if callable(cb_method):
+ cb = cb_method
+ else:
+ raise ValueError(
+ f"Cannot find callback {cb!r} in spider: {spider.name}"
+ )
+ return cb
+
def prepare_request(self, spider, request, opts):
def callback(response, **cb_kwargs):
# memorize first request
if not self.first_response:
self.first_response = response
- # determine real callback
- cb = response.meta["_callback"]
- if not cb:
- if opts.callback:
- cb = opts.callback
- elif opts.rules and self.first_response == response:
- cb = self.get_callback_from_rules(spider, response)
-
- if not cb:
- logger.error(
- "Cannot find a rule that matches %(url)r in spider: %(spider)s",
- {"url": response.url, "spider": spider.name},
- )
- return
- else:
- cb = "parse"
-
- if not callable(cb):
- cb_method = getattr(spider, cb, None)
- if callable(cb_method):
- cb = cb_method
- else:
- logger.error(
- "Cannot find callback %(callback)r in spider: %(spider)s",
- {"callback": cb, "spider": spider.name},
- )
- return
+ cb = self._get_callback(spider=spider, opts=opts, response=response)
# parse items and requests
depth = response.meta["_depth"]
@@ -303,6 +305,9 @@ class Command(BaseRunSpiderCommand):
request.meta["_depth"] = 1
request.meta["_callback"] = request.callback
+ if not request.callback and not opts.rules:
+ cb = self._get_callback(spider=spider, opts=opts)
+ functools.update_wrapper(callback, cb)
request.callback = callback
return request
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index dcf1a6dbc..031a0be36 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -103,8 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
) -> Union[Generator, AsyncGenerator]:
def process_sync(iterable: Iterable) -> Generator:
try:
- for r in iterable:
- yield r
+ yield from iterable
except Exception as ex:
exception_result = self._process_spider_exception(
response, spider, Failure(ex), exception_processor_index
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 844d5f759..1db9ace28 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -178,6 +178,48 @@ class Crawler:
assert self.engine
yield maybeDeferred(self.engine.stop)
+ @staticmethod
+ def _get_component(component_class, components):
+ for component in components:
+ if isinstance(component, component_class):
+ return component
+ return None
+
+ def get_addon(self, cls):
+ return self._get_component(cls, self.addons.addons)
+
+ def get_downloader_middleware(self, cls):
+ if not self.engine:
+ raise RuntimeError(
+ "Crawler.get_downloader_middleware() can only be called after "
+ "the crawl engine has been created."
+ )
+ return self._get_component(cls, self.engine.downloader.middleware.middlewares)
+
+ def get_extension(self, cls):
+ if not self.extensions:
+ raise RuntimeError(
+ "Crawler.get_extension() can only be called after the "
+ "extension manager has been created."
+ )
+ return self._get_component(cls, self.extensions.middlewares)
+
+ def get_item_pipeline(self, cls):
+ if not self.engine:
+ raise RuntimeError(
+ "Crawler.get_item_pipeline() can only be called after the "
+ "crawl engine has been created."
+ )
+ return self._get_component(cls, self.engine.scraper.itemproc.middlewares)
+
+ def get_spider_middleware(self, cls):
+ if not self.engine:
+ raise RuntimeError(
+ "Crawler.get_spider_middleware() can only be called after the "
+ "crawl engine has been created."
+ )
+ return self._get_component(cls, self.engine.scraper.spidermw.middlewares)
+
class CrawlerRunner:
"""
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 221967bda..ca766c938 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -128,9 +128,9 @@ class MemoryUsage:
def _send_report(self, rcpts, subject):
"""send notification mail with some additional useful info"""
stats = self.crawler.stats
- s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
- s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n"
- s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n"
+ s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
+ s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
+ s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n"
s += (
"ENGINE STATUS ------------------------------------------------------- \r\n"
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 5c09ab37e..73064ad10 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -8,7 +8,6 @@ import functools
import hashlib
import logging
import mimetypes
-import os
import time
from collections import defaultdict
from contextlib import suppress
@@ -66,7 +65,7 @@ class FSFilesStore:
absolute_path = self._get_filesystem_path(path)
try:
last_modified = absolute_path.stat().st_mtime
- except os.error:
+ except OSError:
return {}
with absolute_path.open("rb") as f:
@@ -340,7 +339,9 @@ class FilesPipeline(MediaPipeline):
DEFAULT_FILES_URLS_FIELD = "file_urls"
DEFAULT_FILES_RESULT_FIELD = "files"
- def __init__(self, store_uri, download_func=None, settings=None):
+ def __init__(
+ self, store_uri: Union[str, PathLike], download_func=None, settings=None
+ ):
store_uri = _to_string(store_uri)
if not store_uri:
raise NotConfigured
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 1bd9832a8..02c4b1361 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -8,7 +8,8 @@ import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
-from typing import Dict, Tuple
+from os import PathLike
+from typing import Dict, Tuple, Union
from itemadapter import ItemAdapter
@@ -53,7 +54,9 @@ class ImagesPipeline(FilesPipeline):
DEFAULT_IMAGES_URLS_FIELD = "image_urls"
DEFAULT_IMAGES_RESULT_FIELD = "images"
- def __init__(self, store_uri, download_func=None, settings=None):
+ def __init__(
+ self, store_uri: Union[str, PathLike], download_func=None, settings=None
+ ):
try:
from PIL import Image
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d6b3585e2..49ab1b5ef 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -260,7 +260,7 @@ REFERER_ENABLED = True
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
-REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6"
+REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL"
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 31e845716..ebb4f5984 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -131,8 +131,7 @@ class CrawlSpider(Spider):
def _handle_failure(self, failure, errback):
if errback:
results = errback(failure) or ()
- for request_or_item in iterate_spider_output(results):
- yield request_or_item
+ yield from iterate_spider_output(results)
def _compile_rules(self):
self._rules = []
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 6afadc577..47827e442 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -58,8 +58,7 @@ class XMLFeedSpider(Spider):
for selector in nodes:
ret = iterate_spider_output(self.parse_node(response, selector))
- for result_item in self.process_results(response, ret):
- yield result_item
+ yield from self.process_results(response, ret)
def _parse(self, response, **kwargs):
if not hasattr(self, "parse_node"):
@@ -133,8 +132,7 @@ class CSVFeedSpider(Spider):
response, self.delimiter, self.headers, quotechar=self.quotechar
):
ret = iterate_spider_output(self.parse_row(response, row))
- for result_item in self.process_results(response, ret):
- yield result_item
+ yield from self.process_results(response, ret)
def _parse(self, response, **kwargs):
if not hasattr(self, "parse_row"):
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index aaf75a519..974665fe0 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -33,8 +33,7 @@ class SitemapSpider(Spider):
attributes, for example, you can filter locs with lastmod greater
than a given date (see docs).
"""
- for entry in entries:
- yield entry
+ yield from entries
def _parse_sitemap(self, response):
if response.url.endswith("/robots.txt"):
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index ecb1e5e5c..b4779e555 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -88,6 +88,5 @@ ROBOTSTXT_OBEY = True
#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
# Set settings whose default value is deprecated to a future-proof value
-REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
FEED_EXPORT_ENCODING = "utf-8"
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 33e1197bb..7ca22fc80 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -59,8 +59,7 @@ def iflatten(x: Iterable) -> Iterable:
Similar to ``.flatten()``, but returns iterator instead"""
for el in x:
if is_listlike(el):
- for el_ in iflatten(el):
- yield el_
+ yield from iflatten(el)
else:
yield el
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 24fcbd85e..db0b44cf4 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -34,9 +34,6 @@ from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
from scrapy.crawler import Crawler
-_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
-_deprecated_fingerprint_cache = WeakKeyDictionary()
-
def _serialize_headers(
headers: Iterable[bytes], request: Request
@@ -44,122 +41,7 @@ def _serialize_headers(
for header in headers:
if header in request.headers:
yield header
- for value in request.headers.getlist(header):
- yield value
-
-
-def request_fingerprint(
- request: Request,
- include_headers: Optional[Iterable[Union[bytes, str]]] = None,
- keep_fragments: bool = False,
-) -> str:
- """
- Return the request fingerprint as an hexadecimal string.
-
- The request fingerprint is a hash that uniquely identifies the resource the
- request points to. For example, take the following two urls:
-
- http://www.example.com/query?id=111&cat=222
- http://www.example.com/query?cat=222&id=111
-
- Even though those are two different URLs both point to the same resource
- and are equivalent (i.e. they should return the same response).
-
- Another example are cookies used to store session ids. Suppose the
- following page is only accessible to authenticated users:
-
- http://www.example.com/members/offers.html
-
- Lots of sites use a cookie to store the session id, which adds a random
- component to the HTTP Request and thus should be ignored when calculating
- the fingerprint.
-
- For this reason, request headers are ignored by default when calculating
- the fingerprint. If you want to include specific headers use the
- include_headers argument, which is a list of Request headers to include.
-
- Also, servers usually ignore fragments in urls when handling requests,
- so they are also ignored by default when calculating the fingerprint.
- If you want to include them, set the keep_fragments argument to True
- (for instance when handling requests with a headless browser).
- """
- if include_headers or keep_fragments:
- message = (
- "Call to deprecated function "
- "scrapy.utils.request.request_fingerprint().\n"
- "\n"
- "If you are using this function in a Scrapy component because you "
- "need a non-default fingerprinting algorithm, and you are OK "
- "with that non-default fingerprinting algorithm being used by "
- "all Scrapy components and not just the one calling this "
- "function, use crawler.request_fingerprinter.fingerprint() "
- "instead in your Scrapy component (you can get the crawler "
- "object from the 'from_crawler' class method), and use the "
- "'REQUEST_FINGERPRINTER_CLASS' setting to configure your "
- "non-default fingerprinting algorithm.\n"
- "\n"
- "Otherwise, consider using the "
- "scrapy.utils.request.fingerprint() function instead.\n"
- "\n"
- "If you switch to 'fingerprint()', or assign the "
- "'REQUEST_FINGERPRINTER_CLASS' setting a class that uses "
- "'fingerprint()', the generated fingerprints will not only be "
- "bytes instead of a string, but they will also be different from "
- "those generated by 'request_fingerprint()'. Before you switch, "
- "make sure that you understand the consequences of this (e.g. "
- "cache invalidation) and are OK with them; otherwise, consider "
- "implementing your own function which returns the same "
- "fingerprints as the deprecated 'request_fingerprint()' function."
- )
- else:
- message = (
- "Call to deprecated function "
- "scrapy.utils.request.request_fingerprint().\n"
- "\n"
- "If you are using this function in a Scrapy component, and you "
- "are OK with users of your component changing the fingerprinting "
- "algorithm through settings, use "
- "crawler.request_fingerprinter.fingerprint() instead in your "
- "Scrapy component (you can get the crawler object from the "
- "'from_crawler' class method).\n"
- "\n"
- "Otherwise, consider using the "
- "scrapy.utils.request.fingerprint() function instead.\n"
- "\n"
- "Either way, the resulting fingerprints will be returned as "
- "bytes, not as a string, and they will also be different from "
- "those generated by 'request_fingerprint()'. Before you switch, "
- "make sure that you understand the consequences of this (e.g. "
- "cache invalidation) and are OK with them; otherwise, consider "
- "implementing your own function which returns the same "
- "fingerprints as the deprecated 'request_fingerprint()' function."
- )
- warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
- processed_include_headers: Optional[Tuple[bytes, ...]] = None
- if include_headers:
- processed_include_headers = tuple(
- to_bytes(h.lower()) for h in sorted(include_headers)
- )
- cache = _deprecated_fingerprint_cache.setdefault(request, {})
- cache_key = (processed_include_headers, keep_fragments)
- if cache_key not in cache:
- fp = hashlib.sha1()
- fp.update(to_bytes(request.method))
- fp.update(
- to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
- )
- fp.update(request.body or b"")
- if processed_include_headers:
- for part in _serialize_headers(processed_include_headers, request):
- fp.update(part)
- cache[cache_key] = fp.hexdigest()
- return cache[cache_key]
-
-
-def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes:
- with warnings.catch_warnings():
- warnings.simplefilter("ignore")
- return bytes.fromhex(request_fingerprint(*args, **kwargs))
+ yield from request.headers.getlist(header)
_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
@@ -259,33 +141,15 @@ class RequestFingerprinter:
"REQUEST_FINGERPRINTER_IMPLEMENTATION"
)
else:
- implementation = "2.6"
- if implementation == "2.6":
+ implementation = "SENTINEL"
+
+ if implementation != "SENTINEL":
message = (
- "'2.6' is a deprecated value for the "
- "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n"
- "\n"
- "It is also the default value. In other words, it is normal "
- "to get this warning if you have not defined a value for the "
- "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so "
- "for backward compatibility reasons, but it will change in a "
- "future version of Scrapy.\n"
- "\n"
- "See the documentation of the "
- "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for "
- "information on how to handle this deprecation."
+ "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n"
+ "And it will be removed in future version of Scrapy."
)
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
- self._fingerprint = _request_fingerprint_as_bytes
- elif implementation == "2.7":
- self._fingerprint = fingerprint
- else:
- raise ValueError(
- f"Got an invalid value on setting "
- f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': "
- f"{implementation!r}. Valid values are '2.6' (deprecated) "
- f"and '2.7'."
- )
+ self._fingerprint = fingerprint
def fingerprint(self, request: Request) -> bytes:
return self._fingerprint(request)
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 709e0b00d..7a8c5c859 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -86,8 +86,6 @@ def get_crawler(
# Set by default settings that prevent deprecation warnings.
settings: Dict[str, Any] = {}
- if prevent_warnings:
- settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7"
settings.update(settings_dict or {})
runner = CrawlerRunner(settings)
crawler = runner.create_crawler(spidercls or TestSpider)
diff --git a/tests/spiders.py b/tests/spiders.py
index f29dea2a1..3df153a12 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -301,8 +301,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
def parse(self, response):
self.seedsseen.append(response.meta.get("seed"))
- for req in super().parse(response):
- yield req
+ yield from super().parse(response)
class SingleRequestSpider(MetaSpider):
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 129ef0121..592494aba 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -16,11 +16,11 @@ import scrapy
class CheckSpider(scrapy.Spider):
name = '{self.spider_name}'
- start_urls = ['http://toscrape.com']
+ start_urls = ['data:,']
def parse(self, response, **cb_kwargs):
\"\"\"
- @url http://toscrape.com
+ @url data:,
{contracts}
\"\"\"
{parse_def}
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 037333c03..9356d6b79 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -78,9 +78,21 @@ class AsyncDefAsyncioGenExcSpider(scrapy.Spider):
if i > 5:
raise ValueError("Stopping the processing")
+class CallbackSignatureDownloaderMiddleware:
+ def process_request(self, request, spider):
+ from inspect import signature
+ spider.logger.debug(f"request.callback signature: {{signature(request.callback)}}")
+
+
class MySpider(scrapy.Spider):
name = '{self.spider_name}'
+ custom_settings = {{
+ "DOWNLOADER_MIDDLEWARES": {{
+ CallbackSignatureDownloaderMiddleware: 0,
+ }}
+ }}
+
def parse(self, response):
if getattr(self, 'test_arg', None):
self.logger.debug('It Works!')
@@ -220,7 +232,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
self.url("/html"),
]
)
- self.assertIn("DEBUG: It Works!", _textmode(stderr))
+ log = _textmode(stderr)
+ self.assertIn("DEBUG: It Works!", log)
+ self.assertIn(
+ "DEBUG: request.callback signature: (response, foo=None, key=None)", log
+ )
@defer.inlineCallbacks
def test_request_without_meta(self):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 92bd5f38f..989208694 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -12,12 +12,13 @@ import pytest
from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from pytest import mark, raises
-from twisted.internet import defer
+from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.trial import unittest
from w3lib import __version__ as w3lib_version
from zope.interface.exceptions import MultipleInvalid
import scrapy
+from scrapy import Spider
from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions import telnet
@@ -29,6 +30,19 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer, get_mockserver_env
+# To prevent warnings.
+BASE_SETTINGS = {
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+}
+
+
+def get_raw_crawler(spidercls=None, settings_dict=None):
+ """get_crawler alternative that only calls the __init__ method of the
+ crawler."""
+ settings = Settings()
+ settings.setdict(settings_dict or {})
+ return Crawler(spidercls or DefaultSpider, settings)
+
class BaseCrawlerTest(unittest.TestCase):
def assertOptionIsDefault(self, settings, key):
@@ -39,7 +53,7 @@ class BaseCrawlerTest(unittest.TestCase):
class CrawlerTestCase(BaseCrawlerTest):
def test_populate_spidercls_settings(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
- project_settings = {"TEST1": "project", "TEST3": "project"}
+ project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"}
class CustomSettingsSpider(DefaultSpider):
custom_settings = spider_settings
@@ -71,9 +85,9 @@ class CrawlerTestCase(BaseCrawlerTest):
with raises(ValueError):
Crawler(DefaultSpider())
- @defer.inlineCallbacks
+ @inlineCallbacks
def test_crawler_crawl_twice_deprecated(self):
- crawler = Crawler(NoRequestsSpider)
+ crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
yield crawler.crawl()
with pytest.warns(
ScrapyDeprecationWarning,
@@ -81,6 +95,358 @@ class CrawlerTestCase(BaseCrawlerTest):
):
yield crawler.crawl()
+ def test_get_addon(self):
+ class ParentAddon:
+ pass
+
+ class TrackingAddon(ParentAddon):
+ instances = []
+
+ def __init__(self):
+ TrackingAddon.instances.append(self)
+
+ def update_settings(self, settings):
+ pass
+
+ settings = {
+ **BASE_SETTINGS,
+ "ADDONS": {
+ TrackingAddon: 0,
+ },
+ }
+ crawler = get_crawler(settings_dict=settings)
+ self.assertEqual(len(TrackingAddon.instances), 1)
+ expected = TrackingAddon.instances[-1]
+
+ addon = crawler.get_addon(TrackingAddon)
+ self.assertEqual(addon, expected)
+
+ addon = crawler.get_addon(DefaultSpider)
+ self.assertIsNone(addon)
+
+ addon = crawler.get_addon(ParentAddon)
+ self.assertEqual(addon, expected)
+
+ class ChildAddon(TrackingAddon):
+ pass
+
+ addon = crawler.get_addon(ChildAddon)
+ self.assertIsNone(addon)
+
+ @inlineCallbacks
+ def test_get_downloader_middleware(self):
+ class ParentDownloaderMiddleware:
+ pass
+
+ class TrackingDownloaderMiddleware(ParentDownloaderMiddleware):
+ instances = []
+
+ def __init__(self):
+ TrackingDownloaderMiddleware.instances.append(self)
+
+ class MySpider(Spider):
+ name = "myspider"
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler=crawler)
+
+ def __init__(self, crawler):
+ self.crawler = crawler
+
+ def start_requests(self):
+ MySpider.result = crawler.get_downloader_middleware(MySpider.cls)
+ return
+ yield
+
+ settings = {
+ **BASE_SETTINGS,
+ "DOWNLOADER_MIDDLEWARES": {
+ TrackingDownloaderMiddleware: 0,
+ },
+ }
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = TrackingDownloaderMiddleware
+ yield crawler.crawl()
+ self.assertEqual(len(TrackingDownloaderMiddleware.instances), 1)
+ self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1])
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = DefaultSpider
+ yield crawler.crawl()
+ self.assertIsNone(MySpider.result)
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = ParentDownloaderMiddleware
+ yield crawler.crawl()
+ self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1])
+
+ class ChildDownloaderMiddleware(TrackingDownloaderMiddleware):
+ pass
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = ChildDownloaderMiddleware
+ yield crawler.crawl()
+ self.assertIsNone(MySpider.result)
+
+ def test_get_downloader_middleware_not_crawling(self):
+ crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+ self.assertRaises(
+ RuntimeError, crawler.get_downloader_middleware, DefaultSpider
+ )
+
+ @inlineCallbacks
+ def test_get_downloader_middleware_no_engine(self):
+ class MySpider(Spider):
+ name = "myspider"
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ try:
+ crawler.get_downloader_middleware(DefaultSpider)
+ except Exception as e:
+ MySpider.result = e
+ raise
+
+ crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+ with raises(RuntimeError):
+ yield crawler.crawl()
+
+ @inlineCallbacks
+ def test_get_extension(self):
+ class ParentExtension:
+ pass
+
+ class TrackingExtension(ParentExtension):
+ instances = []
+
+ def __init__(self):
+ TrackingExtension.instances.append(self)
+
+ class MySpider(Spider):
+ name = "myspider"
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler=crawler)
+
+ def __init__(self, crawler):
+ self.crawler = crawler
+
+ def start_requests(self):
+ MySpider.result = crawler.get_extension(MySpider.cls)
+ return
+ yield
+
+ settings = {
+ **BASE_SETTINGS,
+ "EXTENSIONS": {
+ TrackingExtension: 0,
+ },
+ }
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = TrackingExtension
+ yield crawler.crawl()
+ self.assertEqual(len(TrackingExtension.instances), 1)
+ self.assertEqual(MySpider.result, TrackingExtension.instances[-1])
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = DefaultSpider
+ yield crawler.crawl()
+ self.assertIsNone(MySpider.result)
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = ParentExtension
+ yield crawler.crawl()
+ self.assertEqual(MySpider.result, TrackingExtension.instances[-1])
+
+ class ChildExtension(TrackingExtension):
+ pass
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = ChildExtension
+ yield crawler.crawl()
+ self.assertIsNone(MySpider.result)
+
+ def test_get_extension_not_crawling(self):
+ crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+ self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider)
+
+ @inlineCallbacks
+ def test_get_extension_no_engine(self):
+ class MySpider(Spider):
+ name = "myspider"
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ try:
+ crawler.get_extension(DefaultSpider)
+ except Exception as e:
+ MySpider.result = e
+ raise
+
+ crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+ with raises(RuntimeError):
+ yield crawler.crawl()
+
+ @inlineCallbacks
+ def test_get_item_pipeline(self):
+ class ParentItemPipeline:
+ pass
+
+ class TrackingItemPipeline(ParentItemPipeline):
+ instances = []
+
+ def __init__(self):
+ TrackingItemPipeline.instances.append(self)
+
+ class MySpider(Spider):
+ name = "myspider"
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler=crawler)
+
+ def __init__(self, crawler):
+ self.crawler = crawler
+
+ def start_requests(self):
+ MySpider.result = crawler.get_item_pipeline(MySpider.cls)
+ return
+ yield
+
+ settings = {
+ **BASE_SETTINGS,
+ "ITEM_PIPELINES": {
+ TrackingItemPipeline: 0,
+ },
+ }
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = TrackingItemPipeline
+ yield crawler.crawl()
+ self.assertEqual(len(TrackingItemPipeline.instances), 1)
+ self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1])
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = DefaultSpider
+ yield crawler.crawl()
+ self.assertIsNone(MySpider.result)
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = ParentItemPipeline
+ yield crawler.crawl()
+ self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1])
+
+ class ChildItemPipeline(TrackingItemPipeline):
+ pass
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = ChildItemPipeline
+ yield crawler.crawl()
+ self.assertIsNone(MySpider.result)
+
+ def test_get_item_pipeline_not_crawling(self):
+ crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+ self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider)
+
+ @inlineCallbacks
+ def test_get_item_pipeline_no_engine(self):
+ class MySpider(Spider):
+ name = "myspider"
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ try:
+ crawler.get_item_pipeline(DefaultSpider)
+ except Exception as e:
+ MySpider.result = e
+ raise
+
+ crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+ with raises(RuntimeError):
+ yield crawler.crawl()
+
+ @inlineCallbacks
+ def test_get_spider_middleware(self):
+ class ParentSpiderMiddleware:
+ pass
+
+ class TrackingSpiderMiddleware(ParentSpiderMiddleware):
+ instances = []
+
+ def __init__(self):
+ TrackingSpiderMiddleware.instances.append(self)
+
+ class MySpider(Spider):
+ name = "myspider"
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler=crawler)
+
+ def __init__(self, crawler):
+ self.crawler = crawler
+
+ def start_requests(self):
+ MySpider.result = crawler.get_spider_middleware(MySpider.cls)
+ return
+ yield
+
+ settings = {
+ **BASE_SETTINGS,
+ "SPIDER_MIDDLEWARES": {
+ TrackingSpiderMiddleware: 0,
+ },
+ }
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = TrackingSpiderMiddleware
+ yield crawler.crawl()
+ self.assertEqual(len(TrackingSpiderMiddleware.instances), 1)
+ self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1])
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = DefaultSpider
+ yield crawler.crawl()
+ self.assertIsNone(MySpider.result)
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = ParentSpiderMiddleware
+ yield crawler.crawl()
+ self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1])
+
+ class ChildSpiderMiddleware(TrackingSpiderMiddleware):
+ pass
+
+ crawler = get_raw_crawler(MySpider, settings)
+ MySpider.cls = ChildSpiderMiddleware
+ yield crawler.crawl()
+ self.assertIsNone(MySpider.result)
+
+ def test_get_spider_middleware_not_crawling(self):
+ crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
+ self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider)
+
+ @inlineCallbacks
+ def test_get_spider_middleware_no_engine(self):
+ class MySpider(Spider):
+ name = "myspider"
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ try:
+ crawler.get_spider_middleware(DefaultSpider)
+ except Exception as e:
+ MySpider.result = e
+ raise
+
+ crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
+ with raises(RuntimeError):
+ yield crawler.crawl()
+
class SpiderSettingsTestCase(unittest.TestCase):
def test_spider_custom_settings(self):
@@ -223,20 +589,20 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
def _runner(self):
return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"})
- @defer.inlineCallbacks
+ @inlineCallbacks
def test_crawler_runner_bootstrap_successful(self):
runner = self._runner()
yield runner.crawl(NoRequestsSpider)
self.assertFalse(runner.bootstrap_failed)
- @defer.inlineCallbacks
+ @inlineCallbacks
def test_crawler_runner_bootstrap_successful_for_several(self):
runner = self._runner()
yield runner.crawl(NoRequestsSpider)
yield runner.crawl(NoRequestsSpider)
self.assertFalse(runner.bootstrap_failed)
- @defer.inlineCallbacks
+ @inlineCallbacks
def test_crawler_runner_bootstrap_failed(self):
runner = self._runner()
@@ -249,7 +615,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
self.assertTrue(runner.bootstrap_failed)
- @defer.inlineCallbacks
+ @inlineCallbacks
def test_crawler_runner_bootstrap_failed_for_several(self):
runner = self._runner()
@@ -264,7 +630,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
self.assertTrue(runner.bootstrap_failed)
- @defer.inlineCallbacks
+ @inlineCallbacks
def test_crawler_runner_asyncio_enabled_true(self):
if self.reactor_pytest == "asyncio":
CrawlerRunner(
@@ -531,7 +897,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
p.expect_exact("Spider closed (shutdown)")
p.wait()
- @defer.inlineCallbacks
+ @inlineCallbacks
def test_shutdown_forced(self):
from twisted.internet import reactor
@@ -543,7 +909,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
p.kill(sig)
p.expect_exact("shutting down gracefully")
# sending the second signal too fast often causes problems
- d = defer.Deferred()
+ d = Deferred()
reactor.callLater(0.1, d.callback, None)
yield d
p.kill(sig)
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index 502ada6be..b7312bbcd 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -67,7 +67,7 @@ def extension(settings=None):
class TestPeriodicLog(unittest.TestCase):
def test_extension_enabled(self):
- # Expected that settings for this extension loaded succesfully
+ # Expected that settings for this extension loaded successfully
# And on certain conditions - extension raising NotConfigured
# "PERIODIC_LOG_STATS": True -> set to {"enabled": True}
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 89169fd7c..277555608 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -673,8 +673,7 @@ class FeedExportTestBase(ABC, unittest.TestCase):
name = "testspider"
def parse(self, response):
- for item in items:
- yield item
+ yield from items
data = yield self.run_and_export(TestSpider, settings)
return data
@@ -2300,7 +2299,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
content[feed["format"]].append(file.read_bytes())
finally:
self.tearDown()
- defer.returnValue(content)
+ return content
@defer.inlineCallbacks
def assertExportedJsonLines(self, items, rows, settings=None):
@@ -2696,8 +2695,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
name = "testspider"
def parse(self, response):
- for item in items:
- yield item
+ yield from items
with MockServer() as server:
TestSpider.start_urls = [server.url("/")]
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index dc0a82086..46d42e9f6 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -31,6 +31,7 @@ sys.exit(mitmdump())
self.proc = Popen(
[
sys.executable,
+ "-u",
"-c",
script,
"--listen-host",
@@ -46,7 +47,7 @@ sys.exit(mitmdump())
stdout=PIPE,
)
line = self.proc.stdout.readline().decode("utf-8")
- host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)
+ host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1)
address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
return address
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 00da3d485..9ce40f921 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -317,8 +317,7 @@ class CrawlSpiderTest(SpiderTest):
rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
def dummy_process_links(self, links):
- for link in links:
- yield link
+ yield from links
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index d167adbb7..38ca8d950 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -170,8 +170,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
class ProcessSpiderOutputSimpleMiddleware:
def process_spider_output(self, response, result, spider):
- for r in result:
- yield r
+ yield from result
class ProcessSpiderOutputAsyncGenMiddleware:
@@ -182,8 +181,7 @@ class ProcessSpiderOutputAsyncGenMiddleware:
class ProcessSpiderOutputUniversalMiddleware:
def process_spider_output(self, response, result, spider):
- for r in result:
- yield r
+ yield from result
async def process_spider_output_async(self, response, result, spider):
async for r in result:
@@ -324,8 +322,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
class ProcessStartRequestsSimpleMiddleware:
def process_start_requests(self, start_requests, spider):
- for r in start_requests:
- yield r
+ yield from start_requests
class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index bb0ebc2a4..a7d54b565 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -107,8 +107,7 @@ class DeferUtilsTest(unittest.TestCase):
class IterErrbackTest(unittest.TestCase):
def test_iter_errback_good(self):
def itergood():
- for x in range(10):
- yield x
+ yield from range(10)
errors = []
out = list(iter_errback(itergood(), errors.append))
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 3598fa0bb..4a0c34d82 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -524,6 +524,6 @@ class TestHelper(unittest.TestCase):
def _assert_type_and_value(self, a, b, obj):
self.assertTrue(
- type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}"
+ type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}"
)
self.assertEqual(a, b)
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index e6d1abe3f..633077eec 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -2,22 +2,15 @@ import json
import unittest
import warnings
from hashlib import sha1
-from typing import Dict, Mapping, Optional, Tuple, Union
+from typing import Dict, Optional, Tuple, Union
from weakref import WeakKeyDictionary
-import pytest
-from w3lib.url import canonicalize_url
-
from scrapy.http import Request
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.python import to_bytes
from scrapy.utils.request import (
- _deprecated_fingerprint_cache,
_fingerprint_cache,
- _request_fingerprint_as_bytes,
fingerprint,
request_authenticate,
- request_fingerprint,
request_httprepr,
request_to_curl,
)
@@ -233,168 +226,6 @@ class FingerprintTest(unittest.TestCase):
self.assertEqual(actual, expected)
-class RequestFingerprintTest(FingerprintTest):
- function = staticmethod(request_fingerprint)
- cache = _deprecated_fingerprint_cache
- known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
- (
- Request("http://example.org"),
- "b2e5245ef826fd9576c93bd6e392fce3133fab62",
- {},
- ),
- (
- Request("https://example.org"),
- "bd10a0a89ea32cdee77917320f1309b0da87e892",
- {},
- ),
- (
- Request("https://example.org?a"),
- "2fb7d48ae02f04b749f40caa969c0bc3c43204ce",
- {},
- ),
- (
- Request("https://example.org?a=b"),
- "42e5fe149b147476e3f67ad0670c57b4cc57856a",
- {},
- ),
- (
- Request("https://example.org?a=b&a"),
- "d23a9787cb56c6375c2cae4453c5a8c634526942",
- {},
- ),
- (
- Request("https://example.org?a=b&a=c"),
- "9a18a7a8552a9182b7f1e05d33876409e421e5c5",
- {},
- ),
- (
- Request("https://example.org", method="POST"),
- "ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6",
- {},
- ),
- (
- Request("https://example.org", body=b"a"),
- "4bb136e54e715a4ea7a9dd1101831765d33f2d60",
- {},
- ),
- (
- Request("https://example.org", method="POST", body=b"a"),
- "6c6595374a304b293be762f7b7be3f54e9947c65",
- {},
- ),
- (
- Request("https://example.org#a", headers={"A": b"B"}),
- "bd10a0a89ea32cdee77917320f1309b0da87e892",
- {},
- ),
- (
- Request("https://example.org#a", headers={"A": b"B"}),
- "515b633cb3ca502a33a9d8c890e889ec1e425e65",
- {"include_headers": ["A"]},
- ),
- (
- Request("https://example.org#a", headers={"A": b"B"}),
- "505c96e7da675920dfef58725e8c957dfdb38f47",
- {"keep_fragments": True},
- ),
- (
- Request("https://example.org#a", headers={"A": b"B"}),
- "d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da",
- {"include_headers": ["A"], "keep_fragments": True},
- ),
- (
- Request("https://example.org/ab"),
- "4e2870fee58582d6f81755e9b8fdefe3cba0c951",
- {},
- ),
- (
- Request("https://example.org/a", body=b"b"),
- "4e2870fee58582d6f81755e9b8fdefe3cba0c951",
- {},
- ),
- )
-
- def setUp(self) -> None:
- warnings.simplefilter("ignore", ScrapyDeprecationWarning)
-
- def tearDown(self) -> None:
- warnings.simplefilter("default", ScrapyDeprecationWarning)
-
- @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
- def test_part_separation(self):
- super().test_part_separation()
-
-
-class RequestFingerprintDeprecationTest(unittest.TestCase):
- def test_deprecation_default_parameters(self):
- with pytest.warns(ScrapyDeprecationWarning) as warnings:
- request_fingerprint(Request("http://www.example.com"))
- messages = [str(warning.message) for warning in warnings]
- self.assertTrue(
- any("Call to deprecated function" in message for message in messages)
- )
- self.assertFalse(any("non-default" in message for message in messages))
-
- def test_deprecation_non_default_parameters(self):
- with pytest.warns(ScrapyDeprecationWarning) as warnings:
- request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
- messages = [str(warning.message) for warning in warnings]
- self.assertTrue(
- any("Call to deprecated function" in message for message in messages)
- )
- self.assertTrue(any("non-default" in message for message in messages))
-
-
-class RequestFingerprintAsBytesTest(FingerprintTest):
- function = staticmethod(_request_fingerprint_as_bytes)
- cache = _deprecated_fingerprint_cache
- known_hashes = RequestFingerprintTest.known_hashes
-
- def test_caching(self):
- r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
- self.assertEqual(
- self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key])
- )
-
- @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
- def test_part_separation(self):
- super().test_part_separation()
-
- def test_hashes(self):
- actual = [
- self.function(request, **kwargs) for request, _, kwargs in self.known_hashes
- ]
- expected = [
- bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes
- ]
- self.assertEqual(actual, expected)
-
-
-_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
-
-
-def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
- if include_headers:
- include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
- cache = _fingerprint_cache_2_6.setdefault(request, {})
- cache_key = (include_headers, keep_fragments)
- if cache_key not in cache:
- fp = sha1()
- fp.update(to_bytes(request.method))
- fp.update(
- to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
- )
- fp.update(request.body or b"")
- if include_headers:
- for hdr in include_headers:
- if hdr in request.headers:
- fp.update(hdr)
- for v in request.headers.getlist(hdr):
- fp.update(v)
- cache[cache_key] = fp.hexdigest()
- return cache[cache_key]
-
-
REQUEST_OBJECTS_TO_TEST = (
Request("http://www.example.com/"),
Request("http://www.example.com/query?id=111&cat=222"),
@@ -424,94 +255,16 @@ REQUEST_OBJECTS_TO_TEST = (
)
-class BackwardCompatibilityTestCase(unittest.TestCase):
- def test_function_backward_compatibility(self):
- include_headers_to_test = (
- None,
- ["Accept-Language"],
- ["accept-language", "sessionid"],
- ["SESSIONID", "Accept-Language"],
- )
- for request_object in REQUEST_OBJECTS_TO_TEST:
- for include_headers in include_headers_to_test:
- for keep_fragments in (False, True):
- with warnings.catch_warnings():
- warnings.simplefilter("ignore")
- fp = request_fingerprint(
- request_object,
- include_headers=include_headers,
- keep_fragments=keep_fragments,
- )
- old_fp = request_fingerprint_2_6(
- request_object,
- include_headers=include_headers,
- keep_fragments=keep_fragments,
- )
- self.assertEqual(fp, old_fp)
-
- def test_component_backward_compatibility(self):
- for request_object in REQUEST_OBJECTS_TO_TEST:
- with warnings.catch_warnings():
- warnings.simplefilter("ignore")
- crawler = get_crawler(prevent_warnings=False)
- fp = crawler.request_fingerprinter.fingerprint(request_object)
- old_fp = request_fingerprint_2_6(request_object)
- self.assertEqual(fp.hex(), old_fp)
-
- def test_custom_component_backward_compatibility(self):
- """Tests that the backward-compatible request fingerprinting class featured
- in the documentation is indeed backward compatible and does not cause a
- warning to be logged."""
-
- class RequestFingerprinter:
- cache = WeakKeyDictionary()
-
- def fingerprint(self, request):
- if request not in self.cache:
- fp = sha1()
- fp.update(to_bytes(request.method))
- fp.update(to_bytes(canonicalize_url(request.url)))
- fp.update(request.body or b"")
- self.cache[request] = fp.digest()
- return self.cache[request]
-
- for request_object in REQUEST_OBJECTS_TO_TEST:
- with warnings.catch_warnings() as logged_warnings:
- settings = {
- "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
- }
- crawler = get_crawler(settings_dict=settings)
- fp = crawler.request_fingerprinter.fingerprint(request_object)
- old_fp = request_fingerprint_2_6(request_object)
- self.assertEqual(fp.hex(), old_fp)
- self.assertFalse(logged_warnings)
-
-
class RequestFingerprinterTestCase(unittest.TestCase):
def test_default_implementation(self):
- with warnings.catch_warnings(record=True) as logged_warnings:
- crawler = get_crawler(prevent_warnings=False)
+ crawler = get_crawler()
request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
- _request_fingerprint_as_bytes(request),
+ fingerprint(request),
)
- self.assertTrue(logged_warnings)
def test_deprecated_implementation(self):
- settings = {
- "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6",
- }
- with warnings.catch_warnings(record=True) as logged_warnings:
- crawler = get_crawler(settings_dict=settings)
- request = Request("https://example.com")
- self.assertEqual(
- crawler.request_fingerprinter.fingerprint(request),
- _request_fingerprint_as_bytes(request),
- )
- self.assertTrue(logged_warnings)
-
- def test_recommended_implementation(self):
settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
@@ -522,14 +275,7 @@ class RequestFingerprinterTestCase(unittest.TestCase):
crawler.request_fingerprinter.fingerprint(request),
fingerprint(request),
)
- self.assertFalse(logged_warnings)
-
- def test_unknown_implementation(self):
- settings = {
- "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",
- }
- with self.assertRaises(ValueError):
- get_crawler(settings_dict=settings)
+ self.assertTrue(logged_warnings)
class CustomRequestFingerprinterTestCase(unittest.TestCase):
diff --git a/tox.ini b/tox.ini
index c0b1045ea..5d57c3408 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,11 +11,7 @@ minversion = 1.7.0
deps =
-rtests/requirements.txt
# mitmproxy does not support PyPy
- # Python 3.9+ requires mitmproxy >= 5.3.0
- # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
- #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
- # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
- mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
+ mitmproxy; implementation_name != 'pypy'
passenv =
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID
@@ -88,7 +84,7 @@ deps =
xtractmime==0.2.0
-rtests/requirements.txt
- # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
+ # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
# above, hence we do not install it in pinned environments at the moment
setenv =
_SCRAPY_PINNED=true