mirror of https://github.com/scrapy/scrapy.git
137 lines
4.5 KiB
Python
137 lines
4.5 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
from warnings import warn
|
|
|
|
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
from scrapy.utils.job import job_dir
|
|
from scrapy.utils.request import (
|
|
RequestFingerprinter,
|
|
RequestFingerprinterProtocol,
|
|
referer_str,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from twisted.internet.defer import Deferred
|
|
|
|
# typing.Self requires Python 3.11
|
|
from typing_extensions import Self
|
|
|
|
from scrapy.crawler import Crawler
|
|
from scrapy.http.request import Request
|
|
from scrapy.spiders import Spider
|
|
|
|
|
|
class BaseDupeFilter:
|
|
"""Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`)
|
|
that does not filter out any request."""
|
|
|
|
@classmethod
|
|
def from_crawler(cls, crawler: Crawler) -> Self:
|
|
return cls()
|
|
|
|
def request_seen(self, request: Request) -> bool:
|
|
return False
|
|
|
|
def open(self) -> Deferred[None] | None:
|
|
pass
|
|
|
|
def close(self, reason: str) -> Deferred[None] | None:
|
|
pass
|
|
|
|
def log(self, request: Request, spider: Spider) -> None:
|
|
"""Log that a request has been filtered"""
|
|
warn(
|
|
"Calling BaseDupeFilter.log() is deprecated.",
|
|
ScrapyDeprecationWarning,
|
|
stacklevel=2,
|
|
)
|
|
|
|
|
|
class RFPDupeFilter(BaseDupeFilter):
|
|
"""Duplicate request filtering class (:setting:`DUPEFILTER_CLASS`) that
|
|
filters out requests with the canonical
|
|
(:func:`w3lib.url.canonicalize_url`) :attr:`~scrapy.http.Request.url`,
|
|
:attr:`~scrapy.http.Request.method` and :attr:`~scrapy.http.Request.body`.
|
|
|
|
Job directory contents
|
|
======================
|
|
|
|
.. warning:: The files that this class generates in the :ref:`job directory
|
|
<job-dir>` are an implementation detail, and may change without a
|
|
warning in a future version of Scrapy. Do not rely on the following
|
|
information for anything other than debugging purposes.
|
|
|
|
When using :setting:`JOBDIR`, seen fingerprints are tracked in a file named
|
|
``requests.seen`` in the :ref:`job directory <job-dir>`, which contains 1
|
|
request fingerprint per line.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
path: str | None = None,
|
|
debug: bool = False,
|
|
*,
|
|
fingerprinter: RequestFingerprinterProtocol | None = None,
|
|
) -> None:
|
|
self.file = None
|
|
self.fingerprinter: RequestFingerprinterProtocol = (
|
|
fingerprinter or RequestFingerprinter()
|
|
)
|
|
self.fingerprints: set[str] = set()
|
|
self.logdupes = True
|
|
self.debug = debug
|
|
self.logger = logging.getLogger(__name__)
|
|
if path:
|
|
# line-by-line writing, see: https://github.com/scrapy/scrapy/issues/6019
|
|
self.file = Path(path, "requests.seen").open(
|
|
"a+", buffering=1, encoding="utf-8"
|
|
)
|
|
self.file.reconfigure(write_through=True)
|
|
self.file.seek(0)
|
|
self.fingerprints.update(x.rstrip() for x in self.file)
|
|
|
|
@classmethod
|
|
def from_crawler(cls, crawler: Crawler) -> Self:
|
|
debug = crawler.settings.getbool("DUPEFILTER_DEBUG")
|
|
return cls(
|
|
job_dir(crawler.settings),
|
|
debug,
|
|
fingerprinter=crawler.request_fingerprinter,
|
|
)
|
|
|
|
def request_seen(self, request: Request) -> bool:
|
|
fp = self.request_fingerprint(request)
|
|
if fp in self.fingerprints:
|
|
return True
|
|
self.fingerprints.add(fp)
|
|
if self.file:
|
|
self.file.write(fp + "\n")
|
|
return False
|
|
|
|
def request_fingerprint(self, request: Request) -> str:
|
|
"""Returns a string that uniquely identifies the specified request."""
|
|
return self.fingerprinter.fingerprint(request).hex()
|
|
|
|
def close(self, reason: str) -> None:
|
|
if self.file:
|
|
self.file.close()
|
|
|
|
def log(self, request: Request, spider: Spider) -> None:
|
|
if self.debug:
|
|
msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
|
|
args = {"request": request, "referer": referer_str(request)}
|
|
self.logger.debug(msg, args, extra={"spider": spider})
|
|
elif self.logdupes:
|
|
msg = (
|
|
"Filtered duplicate request: %(request)s"
|
|
" - no more duplicates will be shown"
|
|
" (see DUPEFILTER_DEBUG to show all duplicates)"
|
|
)
|
|
self.logger.debug(msg, {"request": request}, extra={"spider": spider})
|
|
self.logdupes = False
|
|
|
|
spider.crawler.stats.inc_value("dupefilter/filtered")
|