mirror of https://github.com/scrapy/scrapy.git
Centralize request fingerprints (#4524)
Co-authored-by: Mikhail Korobov <kmike84@gmail.com>
This commit is contained in:
Adrián Chaves
GitHub
parent
517cbc8d71
commit
c5627af15b
|
|
@ -294,7 +294,9 @@ intersphinx_mapping = {
|
|||
'tox': ('https://tox.readthedocs.io/en/latest', None),
|
||||
'twisted': ('https://twistedmatrix.com/documents/current', None),
|
||||
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
|
||||
'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
|
||||
}
|
||||
intersphinx_disabled_reftypes = []
|
||||
|
||||
|
||||
# Options for sphinx-hoverxref options
|
||||
|
|
|
|||
|
|
@ -1643,7 +1643,7 @@ New features
|
|||
:issue:`4370`)
|
||||
|
||||
* A new ``keep_fragments`` parameter of
|
||||
:func:`scrapy.utils.request.request_fingerprint` allows to generate
|
||||
``scrapy.utils.request.request_fingerprint`` allows to generate
|
||||
different fingerprints for requests with different fragments in their URL
|
||||
(:issue:`4104`)
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
|
|||
:class:`scrapy.Spider` subclass and a
|
||||
:class:`scrapy.settings.Settings` object.
|
||||
|
||||
.. attribute:: request_fingerprinter
|
||||
|
||||
The request fingerprint builder of this crawler.
|
||||
|
||||
This is used from extensions and middlewares to build short, unique
|
||||
identifiers for requests. See :ref:`request-fingerprints`.
|
||||
|
||||
.. attribute:: settings
|
||||
|
||||
The settings manager of this crawler.
|
||||
|
|
|
|||
|
|
@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
|
|||
:param spider: the spider which was closed
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
.. classmethod:: from_crawler(cls, crawler)
|
||||
|
||||
If present, this classmethod is called to create a pipeline instance
|
||||
If present, this class method is called to create a pipeline instance
|
||||
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
|
||||
of the pipeline. Crawler object provides access to all Scrapy core
|
||||
components like settings and signals; it is a way for pipeline to
|
||||
|
|
|
|||
|
|
@ -339,6 +339,7 @@ errors if needed::
|
|||
request = failure.request
|
||||
self.logger.error('TimeoutError on %s', request.url)
|
||||
|
||||
|
||||
.. _errback-cb_kwargs:
|
||||
|
||||
Accessing additional data in errback functions
|
||||
|
|
@ -364,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``::
|
|||
main_url=failure.request.cb_kwargs['main_url'],
|
||||
)
|
||||
|
||||
|
||||
.. _request-fingerprints:
|
||||
|
||||
Request fingerprints
|
||||
--------------------
|
||||
|
||||
There are some aspects of scraping, such as filtering out duplicate requests
|
||||
(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
|
||||
:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
|
||||
unique identifier from a :class:`~scrapy.http.Request` object: a request
|
||||
fingerprint.
|
||||
|
||||
You often do not need to worry about request fingerprints, the default request
|
||||
fingerprinter works for most projects.
|
||||
|
||||
However, there is no universal way to generate a unique identifier from a
|
||||
request, because different situations require comparing requests differently.
|
||||
For example, sometimes you may need to compare URLs case-insensitively, include
|
||||
URL fragments, exclude certain URL query parameters, include some or all
|
||||
headers, etc.
|
||||
|
||||
To change how request fingerprints are built for your requests, use the
|
||||
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
|
||||
|
||||
.. setting:: REQUEST_FINGERPRINTER_CLASS
|
||||
|
||||
REQUEST_FINGERPRINTER_CLASS
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Default: :class:`scrapy.utils.request.RequestFingerprinter`
|
||||
|
||||
A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
|
||||
import path.
|
||||
|
||||
.. autoclass:: scrapy.utils.request.RequestFingerprinter
|
||||
|
||||
|
||||
.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Default: ``'PREVIOUS_VERSION'``
|
||||
|
||||
Determines which request fingerprinting algorithm is used by the default
|
||||
request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
|
||||
|
||||
Possible values are:
|
||||
|
||||
- ``'PREVIOUS_VERSION'`` (default)
|
||||
|
||||
This implementation uses the same request fingerprinting algorithm as
|
||||
Scrapy PREVIOUS_VERSION and earlier versions.
|
||||
|
||||
Even though this is the default value for backward compatibility reasons,
|
||||
it is a deprecated value.
|
||||
|
||||
- ``'VERSION'``
|
||||
|
||||
This implementation was introduced in Scrapy VERSION to fix an issue of the
|
||||
previous implementation.
|
||||
|
||||
New projects should use this value. The :command:`startproject` command
|
||||
sets this value in the generated ``settings.py`` file.
|
||||
|
||||
If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are
|
||||
using Scrapy components where changing the request fingerprinting algorithm
|
||||
would cause undesired results, you need to carefully decide when to change the
|
||||
value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
|
||||
setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request
|
||||
fingerprinting algorithm and does not log this warning (
|
||||
:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a
|
||||
class).
|
||||
|
||||
Scenarios where changing the request fingerprinting algorithm may cause
|
||||
undesired results include, for example, using the HTTP cache middleware (see
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
|
||||
Changing the request fingerprinting algorithm would invalidade the current
|
||||
cache, requiring you to redownload all requests again.
|
||||
|
||||
Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in
|
||||
your settings to switch already to the request fingerprinting implementation
|
||||
that will be the only request fingerprinting implementation available in a
|
||||
future version of Scrapy, and remove the deprecation warning triggered by using
|
||||
the default value (``'PREVIOUS_VERSION'``).
|
||||
|
||||
|
||||
.. _PREVIOUS_VERSION-request-fingerprinter:
|
||||
.. _custom-request-fingerprinter:
|
||||
|
||||
Writing your own request fingerprinter
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
A request fingerprinter is a class that must implement the following method:
|
||||
|
||||
.. method:: fingerprint(self, request)
|
||||
|
||||
Return a :class:`bytes` object that uniquely identifies *request*.
|
||||
|
||||
See also :ref:`request-fingerprint-restrictions`.
|
||||
|
||||
:param request: request to fingerprint
|
||||
:type request: scrapy.http.Request
|
||||
|
||||
Additionally, it may also implement the following methods:
|
||||
|
||||
.. classmethod:: from_crawler(cls, crawler)
|
||||
|
||||
If present, this class method is called to create a request fingerprinter
|
||||
instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
|
||||
new instance of the request fingerprinter.
|
||||
|
||||
*crawler* provides access to all Scrapy core components like settings and
|
||||
signals; it is a way for the request fingerprinter to access them and hook
|
||||
its functionality into Scrapy.
|
||||
|
||||
:param crawler: crawler that uses this request fingerprinter
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
.. classmethod:: from_settings(cls, settings)
|
||||
|
||||
If present, and ``from_crawler`` is not defined, this class method is called
|
||||
to create a request fingerprinter instance from a
|
||||
:class:`~scrapy.settings.Settings` object. It must return a new instance of
|
||||
the request fingerprinter.
|
||||
|
||||
The ``fingerprint`` method of the default request fingerprinter,
|
||||
:class:`scrapy.utils.request.RequestFingerprinter`, uses
|
||||
:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
|
||||
common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well
|
||||
in your ``fingerprint`` method implementation:
|
||||
|
||||
.. autofunction:: scrapy.utils.request.fingerprint
|
||||
|
||||
For example, to take the value of a request header named ``X-ID`` into
|
||||
account::
|
||||
|
||||
# my_project/settings.py
|
||||
REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
|
||||
|
||||
# my_project/utils.py
|
||||
from scrapy.utils.request import fingerprint
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
return fingerprint(request, include_headers=['X-ID'])
|
||||
|
||||
You can also write your own fingerprinting logic from scratch.
|
||||
|
||||
However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure
|
||||
you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
|
||||
|
||||
- Caching saves CPU by ensuring that fingerprints are calculated only once
|
||||
per request, and not once per Scrapy component that needs the fingerprint
|
||||
of a request.
|
||||
|
||||
- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
|
||||
request objects do not stay in memory forever just because you have
|
||||
references to them in your cache dictionary.
|
||||
|
||||
For example, to take into account only the URL of a request, without any prior
|
||||
URL canonicalization or taking the request method or body into account::
|
||||
|
||||
from hashlib import sha1
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.url))
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
If you need to be able to override the request fingerprinting for arbitrary
|
||||
requests from your spider callbacks, you may implement a request fingerprinter
|
||||
that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
|
||||
when available, and then falls back to
|
||||
:func:`~scrapy.utils.request.fingerprint`. For example::
|
||||
|
||||
from scrapy.utils.request import fingerprint
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
if 'fingerprint' in request.meta:
|
||||
return request.meta['fingerprint']
|
||||
return fingerprint(request)
|
||||
|
||||
If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION
|
||||
without using the deprecated ``'PREVIOUS_VERSION'`` value of the
|
||||
:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
|
||||
request fingerprinter::
|
||||
|
||||
from hashlib import sha1
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from scrapy.utils.python import to_bytes
|
||||
from w3lib.url import canonicalize_url
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url)))
|
||||
fp.update(request.body or b'')
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
|
||||
.. _request-fingerprint-restrictions:
|
||||
|
||||
Request fingerprint restrictions
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Scrapy components that use request fingerprints may impose additional
|
||||
restrictions on the format of the fingerprints that your :ref:`request
|
||||
fingerprinter <custom-request-fingerprinter>` generates.
|
||||
|
||||
The following built-in Scrapy components have such restrictions:
|
||||
|
||||
- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
|
||||
value of :setting:`HTTPCACHE_STORAGE`)
|
||||
|
||||
Request fingerprints must be at least 1 byte long.
|
||||
|
||||
Path and filename length limits of the file system of
|
||||
:setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
|
||||
the following directory structure is created:
|
||||
|
||||
- :attr:`Spider.name <scrapy.spiders.Spider.name>`
|
||||
|
||||
- first byte of a request fingerprint as hexadecimal
|
||||
|
||||
- fingerprint as hexadecimal
|
||||
|
||||
- filenames up to 16 characters long
|
||||
|
||||
For example, if a request fingerprint is made of 20 bytes (default),
|
||||
:setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
|
||||
and the name of your spider is ``'my_spider'`` your file system must
|
||||
support a file path like::
|
||||
|
||||
/home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
|
||||
|
||||
- :class:`scrapy.extensions.httpcache.DbmCacheStorage`
|
||||
|
||||
The underlying DBM implementation must support keys as long as twice
|
||||
the number of bytes of a request fingerprint, plus 5. For example,
|
||||
if a request fingerprint is made of 20 bytes (default),
|
||||
45-character-long keys must be supported.
|
||||
|
||||
|
||||
.. _topics-request-meta:
|
||||
|
||||
Request.meta special keys
|
||||
|
|
|
|||
|
|
@ -825,12 +825,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
|
|||
|
||||
The class used to detect and filter duplicate requests.
|
||||
|
||||
The default (``RFPDupeFilter``) filters based on request fingerprint using
|
||||
the ``scrapy.utils.request.request_fingerprint`` function. In order to change
|
||||
the way duplicates are checked you could subclass ``RFPDupeFilter`` and
|
||||
override its ``request_fingerprint`` method. This method should accept
|
||||
scrapy :class:`~scrapy.Request` object and return its fingerprint
|
||||
(a string).
|
||||
The default (``RFPDupeFilter``) filters based on the
|
||||
:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
|
||||
|
||||
You can disable filtering of duplicate requests by setting
|
||||
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ class Crawler:
|
|||
self.spidercls.update_settings(self.settings)
|
||||
|
||||
self.signals = SignalManager(self)
|
||||
|
||||
self.stats = load_object(self.settings['STATS_CLASS'])(self)
|
||||
|
||||
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
|
||||
|
|
@ -71,6 +72,12 @@ class Crawler:
|
|||
lf_cls = load_object(self.settings['LOG_FORMATTER'])
|
||||
self.logformatter = lf_cls.from_crawler(self)
|
||||
|
||||
self.request_fingerprinter = create_instance(
|
||||
load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
|
||||
settings=self.settings,
|
||||
crawler=self,
|
||||
)
|
||||
|
||||
reactor_class = self.settings.get("TWISTED_REACTOR")
|
||||
if init_reactor:
|
||||
# this needs to be done after the spider settings are merged,
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
import logging
|
||||
import os
|
||||
from typing import Optional, Set, Type, TypeVar
|
||||
from warnings import warn
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.request import referer_str, request_fingerprint
|
||||
from scrapy.utils.request import referer_str, RequestFingerprinter
|
||||
|
||||
|
||||
BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
|
||||
|
|
@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
|
|||
class RFPDupeFilter(BaseDupeFilter):
|
||||
"""Request Fingerprint duplicates filter"""
|
||||
|
||||
def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
path: Optional[str] = None,
|
||||
debug: bool = False,
|
||||
*,
|
||||
fingerprinter=None,
|
||||
) -> None:
|
||||
self.file = None
|
||||
self.fingerprinter = fingerprinter or RequestFingerprinter()
|
||||
self.fingerprints: Set[str] = set()
|
||||
self.logdupes = True
|
||||
self.debug = debug
|
||||
|
|
@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
self.fingerprints.update(x.rstrip() for x in self.file)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
|
||||
def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
|
||||
debug = settings.getbool('DUPEFILTER_DEBUG')
|
||||
return cls(job_dir(settings), debug)
|
||||
try:
|
||||
return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
|
||||
except TypeError:
|
||||
warn(
|
||||
"RFPDupeFilter subclasses must either modify their '__init__' "
|
||||
"method to support a 'fingerprinter' parameter or reimplement "
|
||||
"the 'from_settings' class method.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
result = cls(job_dir(settings), debug)
|
||||
result.fingerprinter = fingerprinter
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
try:
|
||||
return cls.from_settings(
|
||||
crawler.settings,
|
||||
fingerprinter=crawler.request_fingerprinter,
|
||||
)
|
||||
except TypeError:
|
||||
warn(
|
||||
"RFPDupeFilter subclasses must either modify their overridden "
|
||||
"'__init__' method and 'from_settings' class method to "
|
||||
"support a 'fingerprinter' parameter, or reimplement the "
|
||||
"'from_crawler' class method.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
result = cls.from_settings(crawler.settings)
|
||||
result.fingerprinter = crawler.request_fingerprinter
|
||||
return result
|
||||
|
||||
def request_seen(self, request: Request) -> bool:
|
||||
fp = self.request_fingerprint(request)
|
||||
|
|
@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
|
|||
return False
|
||||
|
||||
def request_fingerprint(self, request: Request) -> str:
|
||||
return request_fingerprint(request)
|
||||
return self.fingerprinter.fingerprint(request).hex()
|
||||
|
||||
def close(self, reason: str) -> None:
|
||||
if self.file:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes
|
|||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -228,6 +227,8 @@ class DbmCacheStorage:
|
|||
|
||||
logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
|
||||
|
||||
self._fingerprinter = spider.crawler.request_fingerprinter
|
||||
|
||||
def close_spider(self, spider):
|
||||
self.db.close()
|
||||
|
||||
|
|
@ -244,7 +245,7 @@ class DbmCacheStorage:
|
|||
return response
|
||||
|
||||
def store_response(self, spider, request, response):
|
||||
key = self._request_key(request)
|
||||
key = self._fingerprinter.fingerprint(request).hex()
|
||||
data = {
|
||||
'status': response.status,
|
||||
'url': response.url,
|
||||
|
|
@ -255,7 +256,7 @@ class DbmCacheStorage:
|
|||
self.db[f'{key}_time'] = str(time())
|
||||
|
||||
def _read_data(self, spider, request):
|
||||
key = self._request_key(request)
|
||||
key = self._fingerprinter.fingerprint(request).hex()
|
||||
db = self.db
|
||||
tkey = f'{key}_time'
|
||||
if tkey not in db:
|
||||
|
|
@ -267,9 +268,6 @@ class DbmCacheStorage:
|
|||
|
||||
return pickle.loads(db[f'{key}_data'])
|
||||
|
||||
def _request_key(self, request):
|
||||
return request_fingerprint(request)
|
||||
|
||||
|
||||
class FilesystemCacheStorage:
|
||||
|
||||
|
|
@ -283,6 +281,8 @@ class FilesystemCacheStorage:
|
|||
logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
|
||||
extra={'spider': spider})
|
||||
|
||||
self._fingerprinter = spider.crawler.request_fingerprinter
|
||||
|
||||
def close_spider(self, spider):
|
||||
pass
|
||||
|
||||
|
|
@ -329,7 +329,7 @@ class FilesystemCacheStorage:
|
|||
f.write(request.body)
|
||||
|
||||
def _get_request_path(self, spider, request):
|
||||
key = request_fingerprint(request)
|
||||
key = self._fingerprinter.fingerprint(request).hex()
|
||||
return os.path.join(self.cachedir, spider.name, key[0:2], key)
|
||||
|
||||
def _read_meta(self, spider, request):
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from scrapy.settings import Settings
|
|||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.defer import mustbe_deferred, defer_result
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
|
||||
|
|
@ -77,6 +76,7 @@ class MediaPipeline:
|
|||
except AttributeError:
|
||||
pipe = cls()
|
||||
pipe.crawler = crawler
|
||||
pipe._fingerprinter = crawler.request_fingerprinter
|
||||
return pipe
|
||||
|
||||
def open_spider(self, spider):
|
||||
|
|
@ -90,7 +90,7 @@ class MediaPipeline:
|
|||
return dfd.addCallback(self.item_completed, item, info)
|
||||
|
||||
def _process_request(self, request, info, item):
|
||||
fp = request_fingerprint(request)
|
||||
fp = self._fingerprinter.fingerprint(request)
|
||||
cb = request.callback or (lambda _: _)
|
||||
eb = request.errback
|
||||
request.callback = None
|
||||
|
|
|
|||
|
|
@ -246,6 +246,9 @@ REDIRECT_PRIORITY_ADJUST = +2
|
|||
REFERER_ENABLED = True
|
||||
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
|
||||
|
||||
REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION'
|
||||
|
||||
RETRY_ENABLED = True
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
|
||||
|
|
|
|||
|
|
@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True
|
|||
#HTTPCACHE_DIR = 'httpcache'
|
||||
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
||||
|
||||
# Set settings whose default value is deprecated to a future-proof value
|
||||
REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION'
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@ scrapy.http.Request objects
|
|||
"""
|
||||
|
||||
import hashlib
|
||||
from typing import Dict, Iterable, Optional, Tuple, Union
|
||||
import json
|
||||
import warnings
|
||||
from typing import Dict, Iterable, List, Optional, Tuple, Union
|
||||
from urllib.parse import urlunparse
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
|
|
@ -12,13 +14,22 @@ from w3lib.http import basic_auth_header
|
|||
from w3lib.url import canonicalize_url
|
||||
|
||||
from scrapy import Request, Spider
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
|
||||
|
||||
_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
|
||||
_fingerprint_cache = WeakKeyDictionary()
|
||||
_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
|
||||
_deprecated_fingerprint_cache = WeakKeyDictionary()
|
||||
|
||||
|
||||
def _serialize_headers(headers, request):
|
||||
for header in headers:
|
||||
if header in request.headers:
|
||||
yield header
|
||||
for value in request.headers.getlist(header):
|
||||
yield value
|
||||
|
||||
|
||||
def request_fingerprint(
|
||||
|
|
@ -26,6 +37,123 @@ def request_fingerprint(
|
|||
include_headers: Optional[Iterable[Union[bytes, str]]] = None,
|
||||
keep_fragments: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
Return the request fingerprint as an hexadecimal string.
|
||||
|
||||
The request fingerprint is a hash that uniquely identifies the resource the
|
||||
request points to. For example, take the following two urls:
|
||||
|
||||
http://www.example.com/query?id=111&cat=222
|
||||
http://www.example.com/query?cat=222&id=111
|
||||
|
||||
Even though those are two different URLs both point to the same resource
|
||||
and are equivalent (i.e. they should return the same response).
|
||||
|
||||
Another example are cookies used to store session ids. Suppose the
|
||||
following page is only accessible to authenticated users:
|
||||
|
||||
http://www.example.com/members/offers.html
|
||||
|
||||
Lots of sites use a cookie to store the session id, which adds a random
|
||||
component to the HTTP Request and thus should be ignored when calculating
|
||||
the fingerprint.
|
||||
|
||||
For this reason, request headers are ignored by default when calculating
|
||||
the fingerprint. If you want to include specific headers use the
|
||||
include_headers argument, which is a list of Request headers to include.
|
||||
|
||||
Also, servers usually ignore fragments in urls when handling requests,
|
||||
so they are also ignored by default when calculating the fingerprint.
|
||||
If you want to include them, set the keep_fragments argument to True
|
||||
(for instance when handling requests with a headless browser).
|
||||
"""
|
||||
if include_headers or keep_fragments:
|
||||
message = (
|
||||
'Call to deprecated function '
|
||||
'scrapy.utils.request.request_fingerprint().\n'
|
||||
'\n'
|
||||
'If you are using this function in a Scrapy component because you '
|
||||
'need a non-default fingerprinting algorithm, and you are OK '
|
||||
'with that non-default fingerprinting algorithm being used by '
|
||||
'all Scrapy components and not just the one calling this '
|
||||
'function, use crawler.request_fingerprinter.fingerprint() '
|
||||
'instead in your Scrapy component (you can get the crawler '
|
||||
'object from the \'from_crawler\' class method), and use the '
|
||||
'\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your '
|
||||
'non-default fingerprinting algorithm.\n'
|
||||
'\n'
|
||||
'Otherwise, consider using the '
|
||||
'scrapy.utils.request.fingerprint() function instead.\n'
|
||||
'\n'
|
||||
'If you switch to \'fingerprint()\', or assign the '
|
||||
'\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses '
|
||||
'\'fingerprint()\', the generated fingerprints will not only be '
|
||||
'bytes instead of a string, but they will also be different from '
|
||||
'those generated by \'request_fingerprint()\'. Before you switch, '
|
||||
'make sure that you understand the consequences of this (e.g. '
|
||||
'cache invalidation) and are OK with them; otherwise, consider '
|
||||
'implementing your own function which returns the same '
|
||||
'fingerprints as the deprecated \'request_fingerprint()\' function.'
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
'Call to deprecated function '
|
||||
'scrapy.utils.request.request_fingerprint().\n'
|
||||
'\n'
|
||||
'If you are using this function in a Scrapy component, and you '
|
||||
'are OK with users of your component changing the fingerprinting '
|
||||
'algorithm through settings, use '
|
||||
'crawler.request_fingerprinter.fingerprint() instead in your '
|
||||
'Scrapy component (you can get the crawler object from the '
|
||||
'\'from_crawler\' class method).\n'
|
||||
'\n'
|
||||
'Otherwise, consider using the '
|
||||
'scrapy.utils.request.fingerprint() function instead.\n'
|
||||
'\n'
|
||||
'Either way, the resulting fingerprints will be returned as '
|
||||
'bytes, not as a string, and they will also be different from '
|
||||
'those generated by \'request_fingerprint()\'. Before you switch, '
|
||||
'make sure that you understand the consequences of this (e.g. '
|
||||
'cache invalidation) and are OK with them; otherwise, consider '
|
||||
'implementing your own function which returns the same '
|
||||
'fingerprints as the deprecated \'request_fingerprint()\' function.'
|
||||
)
|
||||
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
processed_include_headers: Optional[Tuple[bytes, ...]] = None
|
||||
if include_headers:
|
||||
processed_include_headers = tuple(
|
||||
to_bytes(h.lower()) for h in sorted(include_headers)
|
||||
)
|
||||
cache = _deprecated_fingerprint_cache.setdefault(request, {})
|
||||
cache_key = (processed_include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
fp = hashlib.sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
|
||||
fp.update(request.body or b'')
|
||||
if processed_include_headers:
|
||||
for part in _serialize_headers(processed_include_headers, request):
|
||||
fp.update(part)
|
||||
cache[cache_key] = fp.hexdigest()
|
||||
return cache[cache_key]
|
||||
|
||||
|
||||
def _request_fingerprint_as_bytes(*args, **kwargs):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
return bytes.fromhex(request_fingerprint(*args, **kwargs))
|
||||
|
||||
|
||||
_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
|
||||
_fingerprint_cache = WeakKeyDictionary()
|
||||
|
||||
|
||||
def fingerprint(
|
||||
request: Request,
|
||||
*,
|
||||
include_headers: Optional[Iterable[Union[bytes, str]]] = None,
|
||||
keep_fragments: bool = False,
|
||||
) -> bytes:
|
||||
"""
|
||||
Return the request fingerprint.
|
||||
|
||||
|
|
@ -43,7 +171,7 @@ def request_fingerprint(
|
|||
|
||||
http://www.example.com/members/offers.html
|
||||
|
||||
Lot of sites use a cookie to store the session id, which adds a random
|
||||
Lots of sites use a cookie to store the session id, which adds a random
|
||||
component to the HTTP Request and thus should be ignored when calculating
|
||||
the fingerprint.
|
||||
|
||||
|
|
@ -55,29 +183,96 @@ def request_fingerprint(
|
|||
so they are also ignored by default when calculating the fingerprint.
|
||||
If you want to include them, set the keep_fragments argument to True
|
||||
(for instance when handling requests with a headless browser).
|
||||
|
||||
"""
|
||||
headers: Optional[Tuple[bytes, ...]] = None
|
||||
processed_include_headers: Optional[Tuple[bytes, ...]] = None
|
||||
if include_headers:
|
||||
headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
|
||||
processed_include_headers = tuple(
|
||||
to_bytes(h.lower()) for h in sorted(include_headers)
|
||||
)
|
||||
cache = _fingerprint_cache.setdefault(request, {})
|
||||
cache_key = (headers, keep_fragments)
|
||||
cache_key = (processed_include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
fp = hashlib.sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
|
||||
fp.update(request.body or b'')
|
||||
if headers:
|
||||
for hdr in headers:
|
||||
if hdr in request.headers:
|
||||
fp.update(hdr)
|
||||
for v in request.headers.getlist(hdr):
|
||||
fp.update(v)
|
||||
cache[cache_key] = fp.hexdigest()
|
||||
# To decode bytes reliably (JSON does not support bytes), regardless of
|
||||
# character encoding, we use bytes.hex()
|
||||
headers: Dict[str, List[str]] = {}
|
||||
if processed_include_headers:
|
||||
for header in processed_include_headers:
|
||||
if header in request.headers:
|
||||
headers[header.hex()] = [
|
||||
header_value.hex()
|
||||
for header_value in request.headers.getlist(header)
|
||||
]
|
||||
fingerprint_data = {
|
||||
'method': to_unicode(request.method),
|
||||
'url': canonicalize_url(request.url, keep_fragments=keep_fragments),
|
||||
'body': (request.body or b'').hex(),
|
||||
'headers': headers,
|
||||
}
|
||||
fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
|
||||
cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
|
||||
return cache[cache_key]
|
||||
|
||||
|
||||
def request_authenticate(request: Request, username: str, password: str) -> None:
|
||||
class RequestFingerprinter:
|
||||
"""Default fingerprinter.
|
||||
|
||||
It takes into account a canonical version
|
||||
(:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
|
||||
<scrapy.http.Request.url>` and the values of :attr:`request.method
|
||||
<scrapy.http.Request.method>` and :attr:`request.body
|
||||
<scrapy.http.Request.body>`. It then generates an `SHA1
|
||||
<https://en.wikipedia.org/wiki/SHA-1>`_ hash.
|
||||
|
||||
.. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def __init__(self, crawler=None):
|
||||
if crawler:
|
||||
implementation = crawler.settings.get(
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION'
|
||||
)
|
||||
else:
|
||||
implementation = 'PREVIOUS_VERSION'
|
||||
if implementation == 'PREVIOUS_VERSION':
|
||||
message = (
|
||||
'\'PREVIOUS_VERSION\' is a deprecated value for the '
|
||||
'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
|
||||
'\n'
|
||||
'It is also the default value. In other words, it is normal '
|
||||
'to get this warning if you have not defined a value for the '
|
||||
'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so '
|
||||
'for backward compatibility reasons, but it will change in a '
|
||||
'future version of Scrapy.\n'
|
||||
'\n'
|
||||
'See the documentation of the '
|
||||
'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for '
|
||||
'information on how to handle this deprecation.'
|
||||
)
|
||||
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
self._fingerprint = _request_fingerprint_as_bytes
|
||||
elif implementation == 'VERSION':
|
||||
self._fingerprint = fingerprint
|
||||
else:
|
||||
raise ValueError(
|
||||
f'Got an invalid value on setting '
|
||||
f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
|
||||
f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) '
|
||||
f'and \'VERSION\'.'
|
||||
)
|
||||
|
||||
def fingerprint(self, request):
|
||||
return self._fingerprint(request)
|
||||
|
||||
|
||||
def request_authenticate(
|
||||
request: Request,
|
||||
username: str,
|
||||
password: str,
|
||||
) -> None:
|
||||
"""Authenticate the given request (in place) using the HTTP basic access
|
||||
authentication mechanism (RFC 2617) and the given username and password
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ def get_ftp_content_and_delete(
|
|||
return "".join(ftp_data)
|
||||
|
||||
|
||||
def get_crawler(spidercls=None, settings_dict=None):
|
||||
def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
|
||||
"""Return an unconfigured Crawler object. If settings_dict is given, it
|
||||
will be used to populate the crawler settings with a project level
|
||||
priority.
|
||||
|
|
@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None):
|
|||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.spiders import Spider
|
||||
|
||||
runner = CrawlerRunner(settings_dict)
|
||||
# Set by default settings that prevent deprecation warnings.
|
||||
settings = {}
|
||||
if prevent_warnings:
|
||||
settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION'
|
||||
settings.update(settings_dict or {})
|
||||
runner = CrawlerRunner(settings)
|
||||
return runner.create_crawler(spidercls or Spider)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -104,7 +104,8 @@ class CrawlerLoggingTestCase(unittest.TestCase):
|
|||
custom_settings = {
|
||||
'LOG_LEVEL': 'INFO',
|
||||
'LOG_FILE': log_file,
|
||||
# disable telnet if not available to avoid an extra warning
|
||||
# settings to avoid extra warnings
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
|
||||
'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,16 @@ from scrapy.utils.test import get_crawler
|
|||
from tests.spiders import SimpleSpider
|
||||
|
||||
|
||||
def _get_dupefilter(*, crawler=None, settings=None, open=True):
|
||||
if crawler is None:
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
dupefilter = scheduler.df
|
||||
if open:
|
||||
dupefilter.open()
|
||||
return dupefilter
|
||||
|
||||
|
||||
class FromCrawlerRFPDupeFilter(RFPDupeFilter):
|
||||
|
||||
@classmethod
|
||||
|
|
@ -64,9 +74,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
self.assertEqual(scheduler.df.method, 'n/a')
|
||||
|
||||
def test_filter(self):
|
||||
dupefilter = RFPDupeFilter()
|
||||
dupefilter.open()
|
||||
|
||||
dupefilter = _get_dupefilter()
|
||||
r1 = Request('http://scrapytest.org/1')
|
||||
r2 = Request('http://scrapytest.org/2')
|
||||
r3 = Request('http://scrapytest.org/2')
|
||||
|
|
@ -85,7 +93,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
|
||||
path = tempfile.mkdtemp()
|
||||
try:
|
||||
df = RFPDupeFilter(path)
|
||||
df = _get_dupefilter(settings={'JOBDIR': path}, open=False)
|
||||
try:
|
||||
df.open()
|
||||
assert not df.request_seen(r1)
|
||||
|
|
@ -93,7 +101,8 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
finally:
|
||||
df.close('finished')
|
||||
|
||||
df2 = RFPDupeFilter(path)
|
||||
df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False)
|
||||
assert df != df2
|
||||
try:
|
||||
df2.open()
|
||||
assert df2.request_seen(r1)
|
||||
|
|
@ -109,26 +118,24 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
output of request_seen.
|
||||
|
||||
"""
|
||||
dupefilter = _get_dupefilter()
|
||||
r1 = Request('http://scrapytest.org/index.html')
|
||||
r2 = Request('http://scrapytest.org/INDEX.html')
|
||||
|
||||
dupefilter = RFPDupeFilter()
|
||||
dupefilter.open()
|
||||
|
||||
assert not dupefilter.request_seen(r1)
|
||||
assert not dupefilter.request_seen(r2)
|
||||
|
||||
dupefilter.close('finished')
|
||||
|
||||
class CaseInsensitiveRFPDupeFilter(RFPDupeFilter):
|
||||
class RequestFingerprinter:
|
||||
|
||||
def request_fingerprint(self, request):
|
||||
def fingerprint(self, request):
|
||||
fp = hashlib.sha1()
|
||||
fp.update(to_bytes(request.url.lower()))
|
||||
return fp.hexdigest()
|
||||
return fp.digest()
|
||||
|
||||
case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter()
|
||||
case_insensitive_dupefilter.open()
|
||||
settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter}
|
||||
case_insensitive_dupefilter = _get_dupefilter(settings=settings)
|
||||
|
||||
assert not case_insensitive_dupefilter.request_seen(r1)
|
||||
assert case_insensitive_dupefilter.request_seen(r2)
|
||||
|
|
@ -142,8 +149,10 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
r1 = Request('http://scrapytest.org/1')
|
||||
|
||||
path = tempfile.mkdtemp()
|
||||
crawler = get_crawler(settings_dict={'JOBDIR': path})
|
||||
try:
|
||||
df = RFPDupeFilter(path)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
df = scheduler.df
|
||||
df.open()
|
||||
df.request_seen(r1)
|
||||
df.close('finished')
|
||||
|
|
@ -164,11 +173,8 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
settings = {'DUPEFILTER_DEBUG': False,
|
||||
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
|
||||
dupefilter = scheduler.df
|
||||
dupefilter.open()
|
||||
dupefilter = _get_dupefilter(crawler=crawler)
|
||||
|
||||
r1 = Request('http://scrapytest.org/index.html')
|
||||
r2 = Request('http://scrapytest.org/index.html')
|
||||
|
|
@ -193,11 +199,41 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
|
||||
dupefilter = scheduler.df
|
||||
dupefilter.open()
|
||||
dupefilter = _get_dupefilter(crawler=crawler)
|
||||
|
||||
r1 = Request('http://scrapytest.org/index.html')
|
||||
r2 = Request('http://scrapytest.org/index.html',
|
||||
headers={'Referer': 'http://scrapytest.org/INDEX.html'})
|
||||
|
||||
dupefilter.log(r1, spider)
|
||||
dupefilter.log(r2, spider)
|
||||
|
||||
assert crawler.stats.get_value('dupefilter/filtered') == 2
|
||||
log.check_present(
|
||||
(
|
||||
'scrapy.dupefilters',
|
||||
'DEBUG',
|
||||
'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
|
||||
)
|
||||
)
|
||||
log.check_present(
|
||||
(
|
||||
'scrapy.dupefilters',
|
||||
'DEBUG',
|
||||
'Filtered duplicate request: <GET http://scrapytest.org/index.html>'
|
||||
' (referer: http://scrapytest.org/INDEX.html)'
|
||||
)
|
||||
)
|
||||
|
||||
dupefilter.close('finished')
|
||||
|
||||
def test_log_debug_default_dupefilter(self):
|
||||
with LogCapture() as log:
|
||||
settings = {'DUPEFILTER_DEBUG': True}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
dupefilter = _get_dupefilter(crawler=crawler)
|
||||
|
||||
r1 = Request('http://scrapytest.org/index.html')
|
||||
r2 = Request('http://scrapytest.org/index.html',
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from scrapy.pipelines.files import (
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.utils.test import (
|
||||
assert_gcs_environ,
|
||||
get_crawler,
|
||||
get_ftp_content_and_delete,
|
||||
get_gcs_content_and_delete,
|
||||
skip_if_no_boto,
|
||||
|
|
@ -47,7 +48,9 @@ class FilesPipelineTestCase(unittest.TestCase):
|
|||
|
||||
def setUp(self):
|
||||
self.tempdir = mkdtemp()
|
||||
self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir}))
|
||||
settings_dict = {'FILES_STORE': self.tempdir}
|
||||
crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
|
||||
self.pipeline = FilesPipeline.from_crawler(crawler)
|
||||
self.pipeline.download_func = _mocked_download_func
|
||||
self.pipeline.open_spider(None)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,17 +7,17 @@ from twisted.python.failure import Failure
|
|||
from twisted.internet import reactor
|
||||
from twisted.internet.defer import Deferred, inlineCallbacks
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.pipelines.files import FileException
|
||||
from scrapy.pipelines.images import ImagesPipeline
|
||||
from scrapy.pipelines.media import MediaPipeline
|
||||
from scrapy.pipelines.files import FileException
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.signal import disconnect_all
|
||||
from scrapy import signals
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
try:
|
||||
|
|
@ -39,11 +39,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
|
|||
settings = None
|
||||
|
||||
def setUp(self):
|
||||
self.spider = Spider('media.com')
|
||||
self.pipe = self.pipeline_class(download_func=_mocked_download_func,
|
||||
settings=Settings(self.settings))
|
||||
spider_cls = Spider
|
||||
self.spider = spider_cls('media.com')
|
||||
crawler = get_crawler(spider_cls, self.settings)
|
||||
self.pipe = self.pipeline_class.from_crawler(crawler)
|
||||
self.pipe.download_func = _mocked_download_func
|
||||
self.pipe.open_spider(self.spider)
|
||||
self.info = self.pipe.spiderinfo
|
||||
self.fingerprint = crawler.request_fingerprinter.fingerprint
|
||||
|
||||
def tearDown(self):
|
||||
for name, signal in vars(signals).items():
|
||||
|
|
@ -156,7 +159,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
|
|||
self.assertEqual(failure.value.__context__, def_gen_return_exc)
|
||||
|
||||
# Let's calculate the request fingerprint and fake some runtime data...
|
||||
fp = request_fingerprint(request)
|
||||
fp = self.fingerprint(request)
|
||||
info = self.pipe.spiderinfo
|
||||
info.downloading.add(fp)
|
||||
info.waiting[fp] = []
|
||||
|
|
@ -273,7 +276,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
item = dict(requests=req) # pass a single item
|
||||
new_item = yield self.pipe.process_item(item, self.spider)
|
||||
assert new_item is item
|
||||
assert request_fingerprint(req) in self.info.downloaded
|
||||
self.assertIn(self.fingerprint(req), self.info.downloaded)
|
||||
|
||||
# returns iterable of Requests
|
||||
req1 = Request('http://url1')
|
||||
|
|
@ -281,8 +284,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
item = dict(requests=iter([req1, req2]))
|
||||
new_item = yield self.pipe.process_item(item, self.spider)
|
||||
assert new_item is item
|
||||
assert request_fingerprint(req1) in self.info.downloaded
|
||||
assert request_fingerprint(req2) in self.info.downloaded
|
||||
assert self.fingerprint(req1) in self.info.downloaded
|
||||
assert self.fingerprint(req2) in self.info.downloaded
|
||||
|
||||
@inlineCallbacks
|
||||
def test_results_are_cached_across_multiple_items(self):
|
||||
|
|
@ -298,7 +301,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
item = dict(requests=req2)
|
||||
new_item = yield self.pipe.process_item(item, self.spider)
|
||||
self.assertTrue(new_item is item)
|
||||
self.assertEqual(request_fingerprint(req1), request_fingerprint(req2))
|
||||
self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))
|
||||
self.assertEqual(new_item['results'], [(True, rsp1)])
|
||||
|
||||
@inlineCallbacks
|
||||
|
|
@ -314,7 +317,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
@inlineCallbacks
|
||||
def test_wait_if_request_is_downloading(self):
|
||||
def _check_downloading(response):
|
||||
fp = request_fingerprint(req1)
|
||||
fp = self.fingerprint(req1)
|
||||
self.assertTrue(fp in self.info.downloading)
|
||||
self.assertTrue(fp in self.info.waiting)
|
||||
self.assertTrue(fp not in self.info.downloaded)
|
||||
|
|
@ -351,7 +354,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
self._mockcalled = []
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
|
|
@ -369,19 +372,19 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
|
|||
|
||||
def media_to_download(self, request, info):
|
||||
self._mockcalled.append('media_to_download')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info)
|
||||
return super().media_to_download(request, info)
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
self._mockcalled.append('media_downloaded')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info)
|
||||
return super().media_downloaded(response, request, info)
|
||||
|
||||
def file_downloaded(self, response, request, info):
|
||||
self._mockcalled.append('file_downloaded')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info)
|
||||
return super().file_downloaded(response, request, info)
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
self._mockcalled.append('file_path')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)
|
||||
return super().file_path(request, response, info)
|
||||
|
||||
def thumb_path(self, request, thumb_id, response=None, info=None):
|
||||
self._mockcalled.append('thumb_path')
|
||||
|
|
@ -393,18 +396,20 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
|
|||
|
||||
def image_downloaded(self, response, request, info):
|
||||
self._mockcalled.append('image_downloaded')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info)
|
||||
return super().image_downloaded(response, request, info)
|
||||
|
||||
|
||||
class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
|
||||
skip = skip_pillow
|
||||
|
||||
def setUp(self):
|
||||
self.pipe = MockedMediaPipelineDeprecatedMethods(
|
||||
store_uri='store-uri',
|
||||
download_func=_mocked_download_func,
|
||||
settings=Settings({"IMAGES_THUMBS": {'small': (50, 50)}})
|
||||
)
|
||||
settings_dict = {
|
||||
'IMAGES_STORE': 'store-uri',
|
||||
'IMAGES_THUMBS': {'small': (50, 50)},
|
||||
}
|
||||
crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
|
||||
self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
|
||||
self.pipe.download_func = _mocked_download_func
|
||||
self.pipe.open_spider(None)
|
||||
self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
|
||||
|
||||
|
|
|
|||
|
|
@ -1,73 +1,29 @@
|
|||
import unittest
|
||||
import warnings
|
||||
from hashlib import sha1
|
||||
from typing import Dict, Mapping, Optional, Tuple, Union
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
import pytest
|
||||
from w3lib.url import canonicalize_url
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.request import (
|
||||
_deprecated_fingerprint_cache,
|
||||
_fingerprint_cache,
|
||||
_request_fingerprint_as_bytes,
|
||||
fingerprint,
|
||||
request_authenticate,
|
||||
request_fingerprint,
|
||||
request_httprepr,
|
||||
)
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
class UtilsRequestTest(unittest.TestCase):
|
||||
|
||||
def test_request_fingerprint(self):
|
||||
r1 = Request("http://www.example.com/query?id=111&cat=222")
|
||||
r2 = Request("http://www.example.com/query?cat=222&id=111")
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
|
||||
r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
|
||||
r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
|
||||
self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
|
||||
# make sure caching is working
|
||||
self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)])
|
||||
|
||||
r1 = Request("http://www.example.com/members/offers.html")
|
||||
r2 = Request("http://www.example.com/members/offers.html")
|
||||
r2.headers['SESSIONID'] = b"somehash"
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
|
||||
r1 = Request("http://www.example.com/")
|
||||
r2 = Request("http://www.example.com/")
|
||||
r2.headers['Accept-Language'] = b'en'
|
||||
r3 = Request("http://www.example.com/")
|
||||
r3.headers['Accept-Language'] = b'en'
|
||||
r3.headers['SESSIONID'] = b"somehash"
|
||||
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3))
|
||||
|
||||
self.assertEqual(request_fingerprint(r1),
|
||||
request_fingerprint(r1, include_headers=['Accept-Language']))
|
||||
|
||||
self.assertNotEqual(
|
||||
request_fingerprint(r1),
|
||||
request_fingerprint(r2, include_headers=['Accept-Language']))
|
||||
|
||||
self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
|
||||
request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
|
||||
|
||||
r1 = Request("http://www.example.com/test.html")
|
||||
r2 = Request("http://www.example.com/test.html#fragment")
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True))
|
||||
self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True))
|
||||
self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True))
|
||||
|
||||
r1 = Request("http://www.example.com")
|
||||
r2 = Request("http://www.example.com", method='POST')
|
||||
r3 = Request("http://www.example.com", method='POST', body=b'request body')
|
||||
|
||||
self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3))
|
||||
|
||||
# cached fingerprint must be cleared on request copy
|
||||
r1 = Request("http://www.example.com")
|
||||
fp1 = request_fingerprint(r1)
|
||||
r2 = r1.replace(url="http://www.example.com/other")
|
||||
fp2 = request_fingerprint(r2)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
|
||||
def test_request_authenticate(self):
|
||||
r = Request("http://www.example.com")
|
||||
request_authenticate(r, 'someuser', 'somepass')
|
||||
|
|
@ -93,5 +49,632 @@ class UtilsRequestTest(unittest.TestCase):
|
|||
request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
|
||||
|
||||
|
||||
class FingerprintTest(unittest.TestCase):
|
||||
maxDiff = None
|
||||
|
||||
function = staticmethod(fingerprint)
|
||||
cache: Union[
|
||||
"WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]",
|
||||
"WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]",
|
||||
] = _fingerprint_cache
|
||||
default_cache_key = (None, False)
|
||||
known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
|
||||
(
|
||||
Request("http://example.org"),
|
||||
b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org"),
|
||||
b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a"),
|
||||
b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b"),
|
||||
b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b&a"),
|
||||
b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b&a=c"),
|
||||
b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", method='POST'),
|
||||
b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", body=b'a'),
|
||||
b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", method='POST', body=b'a'),
|
||||
b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.',
|
||||
{'include_headers': ['A']},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f',
|
||||
{'keep_fragments': True},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11',
|
||||
{'include_headers': ['A'], 'keep_fragments': True},
|
||||
),
|
||||
(
|
||||
Request("https://example.org/ab"),
|
||||
b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org/a", body=b'b'),
|
||||
b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5',
|
||||
{},
|
||||
),
|
||||
)
|
||||
|
||||
def test_query_string_key_order(self):
|
||||
r1 = Request("http://www.example.com/query?id=111&cat=222")
|
||||
r2 = Request("http://www.example.com/query?cat=222&id=111")
|
||||
self.assertEqual(self.function(r1), self.function(r1))
|
||||
self.assertEqual(self.function(r1), self.function(r2))
|
||||
|
||||
def test_query_string_key_without_value(self):
|
||||
r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
|
||||
r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
|
||||
self.assertNotEqual(self.function(r1), self.function(r2))
|
||||
|
||||
def test_caching(self):
|
||||
r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
|
||||
self.assertEqual(
|
||||
self.function(r1),
|
||||
self.cache[r1][self.default_cache_key]
|
||||
)
|
||||
|
||||
def test_header(self):
|
||||
r1 = Request("http://www.example.com/members/offers.html")
|
||||
r2 = Request("http://www.example.com/members/offers.html")
|
||||
r2.headers['SESSIONID'] = b"somehash"
|
||||
self.assertEqual(self.function(r1), self.function(r2))
|
||||
|
||||
def test_headers(self):
|
||||
r1 = Request("http://www.example.com/")
|
||||
r2 = Request("http://www.example.com/")
|
||||
r2.headers['Accept-Language'] = b'en'
|
||||
r3 = Request("http://www.example.com/")
|
||||
r3.headers['Accept-Language'] = b'en'
|
||||
r3.headers['SESSIONID'] = b"somehash"
|
||||
|
||||
self.assertEqual(self.function(r1), self.function(r2), self.function(r3))
|
||||
|
||||
self.assertEqual(self.function(r1),
|
||||
self.function(r1, include_headers=['Accept-Language']))
|
||||
|
||||
self.assertNotEqual(
|
||||
self.function(r1),
|
||||
self.function(r2, include_headers=['Accept-Language']))
|
||||
|
||||
self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']),
|
||||
self.function(r3, include_headers=['SESSIONID', 'Accept-Language']))
|
||||
|
||||
def test_fragment(self):
|
||||
r1 = Request("http://www.example.com/test.html")
|
||||
r2 = Request("http://www.example.com/test.html#fragment")
|
||||
self.assertEqual(self.function(r1), self.function(r2))
|
||||
self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True))
|
||||
self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True))
|
||||
self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True))
|
||||
|
||||
def test_method_and_body(self):
|
||||
r1 = Request("http://www.example.com")
|
||||
r2 = Request("http://www.example.com", method='POST')
|
||||
r3 = Request("http://www.example.com", method='POST', body=b'request body')
|
||||
|
||||
self.assertNotEqual(self.function(r1), self.function(r2))
|
||||
self.assertNotEqual(self.function(r2), self.function(r3))
|
||||
|
||||
def test_request_replace(self):
|
||||
# cached fingerprint must be cleared on request copy
|
||||
r1 = Request("http://www.example.com")
|
||||
fp1 = self.function(r1)
|
||||
r2 = r1.replace(url="http://www.example.com/other")
|
||||
fp2 = self.function(r2)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
|
||||
def test_part_separation(self):
|
||||
# An old implementation used to serialize request data in a way that
|
||||
# would put the body right after the URL.
|
||||
r1 = Request("http://www.example.com/foo")
|
||||
fp1 = self.function(r1)
|
||||
r2 = Request("http://www.example.com/f", body=b'oo')
|
||||
fp2 = self.function(r2)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
|
||||
def test_hashes(self):
|
||||
"""Test hardcoded hashes, to make sure future changes to not introduce
|
||||
backward incompatibilities."""
|
||||
actual = [
|
||||
self.function(request, **kwargs)
|
||||
for request, _, kwargs in self.known_hashes
|
||||
]
|
||||
expected = [
|
||||
_fingerprint
|
||||
for _, _fingerprint, _ in self.known_hashes
|
||||
]
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
|
||||
class RequestFingerprintTest(FingerprintTest):
|
||||
function = staticmethod(request_fingerprint)
|
||||
cache = _deprecated_fingerprint_cache
|
||||
known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
|
||||
(
|
||||
Request("http://example.org"),
|
||||
'b2e5245ef826fd9576c93bd6e392fce3133fab62',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org"),
|
||||
'bd10a0a89ea32cdee77917320f1309b0da87e892',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a"),
|
||||
'2fb7d48ae02f04b749f40caa969c0bc3c43204ce',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b"),
|
||||
'42e5fe149b147476e3f67ad0670c57b4cc57856a',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b&a"),
|
||||
'd23a9787cb56c6375c2cae4453c5a8c634526942',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org?a=b&a=c"),
|
||||
'9a18a7a8552a9182b7f1e05d33876409e421e5c5',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", method='POST'),
|
||||
'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", body=b'a'),
|
||||
'4bb136e54e715a4ea7a9dd1101831765d33f2d60',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org", method='POST', body=b'a'),
|
||||
'6c6595374a304b293be762f7b7be3f54e9947c65',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
'bd10a0a89ea32cdee77917320f1309b0da87e892',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
'515b633cb3ca502a33a9d8c890e889ec1e425e65',
|
||||
{'include_headers': ['A']},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
'505c96e7da675920dfef58725e8c957dfdb38f47',
|
||||
{'keep_fragments': True},
|
||||
),
|
||||
(
|
||||
Request("https://example.org#a", headers={'A': b'B'}),
|
||||
'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da',
|
||||
{'include_headers': ['A'], 'keep_fragments': True},
|
||||
),
|
||||
(
|
||||
Request("https://example.org/ab"),
|
||||
'4e2870fee58582d6f81755e9b8fdefe3cba0c951',
|
||||
{},
|
||||
),
|
||||
(
|
||||
Request("https://example.org/a", body=b'b'),
|
||||
'4e2870fee58582d6f81755e9b8fdefe3cba0c951',
|
||||
{},
|
||||
),
|
||||
)
|
||||
|
||||
@pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
|
||||
def test_part_separation(self):
|
||||
super().test_part_separation()
|
||||
|
||||
def test_deprecation_default_parameters(self):
|
||||
with pytest.warns(ScrapyDeprecationWarning) as warnings:
|
||||
self.function(Request("http://www.example.com"))
|
||||
messages = [str(warning.message) for warning in warnings]
|
||||
self.assertTrue(
|
||||
any(
|
||||
'Call to deprecated function' in message
|
||||
for message in messages
|
||||
)
|
||||
)
|
||||
self.assertFalse(any('non-default' in message for message in messages))
|
||||
|
||||
def test_deprecation_non_default_parameters(self):
|
||||
with pytest.warns(ScrapyDeprecationWarning) as warnings:
|
||||
self.function(Request("http://www.example.com"), keep_fragments=True)
|
||||
messages = [str(warning.message) for warning in warnings]
|
||||
self.assertTrue(
|
||||
any(
|
||||
'Call to deprecated function' in message
|
||||
for message in messages
|
||||
)
|
||||
)
|
||||
self.assertTrue(any('non-default' in message for message in messages))
|
||||
|
||||
|
||||
class RequestFingerprintAsBytesTest(FingerprintTest):
|
||||
function = staticmethod(_request_fingerprint_as_bytes)
|
||||
cache = _deprecated_fingerprint_cache
|
||||
known_hashes = RequestFingerprintTest.known_hashes
|
||||
|
||||
def test_caching(self):
|
||||
r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
|
||||
self.assertEqual(
|
||||
self.function(r1),
|
||||
bytes.fromhex(self.cache[r1][self.default_cache_key])
|
||||
)
|
||||
|
||||
@pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
|
||||
def test_part_separation(self):
|
||||
super().test_part_separation()
|
||||
|
||||
def test_hashes(self):
|
||||
actual = [
|
||||
self.function(request, **kwargs)
|
||||
for request, _, kwargs in self.known_hashes
|
||||
]
|
||||
expected = [
|
||||
bytes.fromhex(_fingerprint)
|
||||
for _, _fingerprint, _ in self.known_hashes
|
||||
]
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
|
||||
_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
|
||||
|
||||
|
||||
def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
|
||||
if include_headers:
|
||||
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
|
||||
cache = _fingerprint_cache_2_6.setdefault(request, {})
|
||||
cache_key = (include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
|
||||
fp.update(request.body or b'')
|
||||
if include_headers:
|
||||
for hdr in include_headers:
|
||||
if hdr in request.headers:
|
||||
fp.update(hdr)
|
||||
for v in request.headers.getlist(hdr):
|
||||
fp.update(v)
|
||||
cache[cache_key] = fp.hexdigest()
|
||||
return cache[cache_key]
|
||||
|
||||
|
||||
REQUEST_OBJECTS_TO_TEST = (
|
||||
Request("http://www.example.com/"),
|
||||
Request("http://www.example.com/query?id=111&cat=222"),
|
||||
Request("http://www.example.com/query?cat=222&id=111"),
|
||||
Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'),
|
||||
Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'),
|
||||
Request("http://www.example.com/members/offers.html"),
|
||||
Request(
|
||||
"http://www.example.com/members/offers.html",
|
||||
headers={'SESSIONID': b"somehash"},
|
||||
),
|
||||
Request(
|
||||
"http://www.example.com/",
|
||||
headers={'Accept-Language': b"en"},
|
||||
),
|
||||
Request(
|
||||
"http://www.example.com/",
|
||||
headers={
|
||||
'Accept-Language': b"en",
|
||||
'SESSIONID': b"somehash",
|
||||
},
|
||||
),
|
||||
Request("http://www.example.com/test.html"),
|
||||
Request("http://www.example.com/test.html#fragment"),
|
||||
Request("http://www.example.com", method='POST'),
|
||||
Request("http://www.example.com", method='POST', body=b'request body'),
|
||||
)
|
||||
|
||||
|
||||
class BackwardCompatibilityTestCase(unittest.TestCase):
|
||||
|
||||
def test_function_backward_compatibility(self):
|
||||
include_headers_to_test = (
|
||||
None,
|
||||
['Accept-Language'],
|
||||
['accept-language', 'sessionid'],
|
||||
['SESSIONID', 'Accept-Language'],
|
||||
)
|
||||
for request_object in REQUEST_OBJECTS_TO_TEST:
|
||||
for include_headers in include_headers_to_test:
|
||||
for keep_fragments in (False, True):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
fp = request_fingerprint(
|
||||
request_object,
|
||||
include_headers=include_headers,
|
||||
keep_fragments=keep_fragments,
|
||||
)
|
||||
old_fp = request_fingerprint_2_6(
|
||||
request_object,
|
||||
include_headers=include_headers,
|
||||
keep_fragments=keep_fragments,
|
||||
)
|
||||
self.assertEqual(fp, old_fp)
|
||||
|
||||
def test_component_backward_compatibility(self):
|
||||
for request_object in REQUEST_OBJECTS_TO_TEST:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
crawler = get_crawler(prevent_warnings=False)
|
||||
fp = crawler.request_fingerprinter.fingerprint(request_object)
|
||||
old_fp = request_fingerprint_2_6(request_object)
|
||||
self.assertEqual(fp.hex(), old_fp)
|
||||
|
||||
def test_custom_component_backward_compatibility(self):
|
||||
"""Tests that the backward-compatible request fingerprinting class featured
|
||||
in the documentation is indeed backward compatible and does not cause a
|
||||
warning to be logged."""
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url)))
|
||||
fp.update(request.body or b'')
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
for request_object in REQUEST_OBJECTS_TO_TEST:
|
||||
with warnings.catch_warnings() as logged_warnings:
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
fp = crawler.request_fingerprinter.fingerprint(request_object)
|
||||
old_fp = request_fingerprint_2_6(request_object)
|
||||
self.assertEqual(fp.hex(), old_fp)
|
||||
self.assertFalse(logged_warnings)
|
||||
|
||||
|
||||
class RequestFingerprinterTestCase(unittest.TestCase):
|
||||
|
||||
def test_default_implementation(self):
|
||||
with warnings.catch_warnings(record=True) as logged_warnings:
|
||||
crawler = get_crawler(prevent_warnings=False)
|
||||
request = Request('https://example.com')
|
||||
self.assertEqual(
|
||||
crawler.request_fingerprinter.fingerprint(request),
|
||||
_request_fingerprint_as_bytes(request),
|
||||
)
|
||||
self.assertTrue(logged_warnings)
|
||||
|
||||
def test_deprecated_implementation(self):
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION',
|
||||
}
|
||||
with warnings.catch_warnings(record=True) as logged_warnings:
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
request = Request('https://example.com')
|
||||
self.assertEqual(
|
||||
crawler.request_fingerprinter.fingerprint(request),
|
||||
_request_fingerprint_as_bytes(request),
|
||||
)
|
||||
self.assertTrue(logged_warnings)
|
||||
|
||||
def test_recommended_implementation(self):
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
|
||||
}
|
||||
with warnings.catch_warnings(record=True) as logged_warnings:
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
request = Request('https://example.com')
|
||||
self.assertEqual(
|
||||
crawler.request_fingerprinter.fingerprint(request),
|
||||
fingerprint(request),
|
||||
)
|
||||
self.assertFalse(logged_warnings)
|
||||
|
||||
def test_unknown_implementation(self):
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5',
|
||||
}
|
||||
with self.assertRaises(ValueError):
|
||||
get_crawler(settings_dict=settings)
|
||||
|
||||
|
||||
class CustomRequestFingerprinterTestCase(unittest.TestCase):
|
||||
|
||||
def test_include_headers(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
return fingerprint(request, include_headers=['X-ID'])
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
r1 = Request("http://www.example.com", headers={'X-ID': '1'})
|
||||
fp1 = crawler.request_fingerprinter.fingerprint(r1)
|
||||
r2 = Request("http://www.example.com", headers={'X-ID': '2'})
|
||||
fp2 = crawler.request_fingerprinter.fingerprint(r2)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
|
||||
def test_dont_canonicalize(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
def fingerprint(self, request):
|
||||
if request not in self.cache:
|
||||
fp = sha1()
|
||||
fp.update(to_bytes(request.url))
|
||||
self.cache[request] = fp.digest()
|
||||
return self.cache[request]
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
r1 = Request("http://www.example.com?a=1&a=2")
|
||||
fp1 = crawler.request_fingerprinter.fingerprint(r1)
|
||||
r2 = Request("http://www.example.com?a=2&a=1")
|
||||
fp2 = crawler.request_fingerprinter.fingerprint(r2)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
|
||||
def test_meta(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
def fingerprint(self, request):
|
||||
if 'fingerprint' in request.meta:
|
||||
return request.meta['fingerprint']
|
||||
return fingerprint(request)
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
r1 = Request("http://www.example.com")
|
||||
fp1 = crawler.request_fingerprinter.fingerprint(r1)
|
||||
r2 = Request("http://www.example.com", meta={'fingerprint': 'a'})
|
||||
fp2 = crawler.request_fingerprinter.fingerprint(r2)
|
||||
r3 = Request("http://www.example.com", meta={'fingerprint': 'a'})
|
||||
fp3 = crawler.request_fingerprinter.fingerprint(r3)
|
||||
r4 = Request("http://www.example.com", meta={'fingerprint': 'b'})
|
||||
fp4 = crawler.request_fingerprinter.fingerprint(r4)
|
||||
self.assertNotEqual(fp1, fp2)
|
||||
self.assertNotEqual(fp1, fp4)
|
||||
self.assertNotEqual(fp2, fp4)
|
||||
self.assertEqual(fp2, fp3)
|
||||
|
||||
def test_from_crawler(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def __init__(self, crawler):
|
||||
self._fingerprint = crawler.settings['FINGERPRINT']
|
||||
|
||||
def fingerprint(self, request):
|
||||
return self._fingerprint
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
'FINGERPRINT': b'fingerprint',
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
request = Request("http://www.example.com")
|
||||
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
||||
self.assertEqual(fingerprint, settings['FINGERPRINT'])
|
||||
|
||||
def test_from_settings(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
return cls(settings)
|
||||
|
||||
def __init__(self, settings):
|
||||
self._fingerprint = settings['FINGERPRINT']
|
||||
|
||||
def fingerprint(self, request):
|
||||
return self._fingerprint
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
'FINGERPRINT': b'fingerprint',
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
request = Request("http://www.example.com")
|
||||
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
||||
self.assertEqual(fingerprint, settings['FINGERPRINT'])
|
||||
|
||||
def test_from_crawler_and_settings(self):
|
||||
|
||||
class RequestFingerprinter:
|
||||
|
||||
# This method is ignored due to the presence of from_crawler
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
return cls(settings)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler)
|
||||
|
||||
def __init__(self, crawler):
|
||||
self._fingerprint = crawler.settings['FINGERPRINT']
|
||||
|
||||
def fingerprint(self, request):
|
||||
return self._fingerprint
|
||||
|
||||
settings = {
|
||||
'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
|
||||
'FINGERPRINT': b'fingerprint',
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
|
||||
request = Request("http://www.example.com")
|
||||
fingerprint = crawler.request_fingerprinter.fingerprint(request)
|
||||
self.assertEqual(fingerprint, settings['FINGERPRINT'])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue