mirror of https://github.com/scrapy/scrapy.git
72 lines
2.1 KiB
Python
72 lines
2.1 KiB
Python
from __future__ import print_function
|
|
import os
|
|
|
|
from scrapy import log
|
|
from scrapy.utils.job import job_dir
|
|
from scrapy.utils.request import request_fingerprint
|
|
|
|
|
|
class BaseDupeFilter(object):
|
|
|
|
@classmethod
|
|
def from_settings(cls, settings):
|
|
return cls()
|
|
|
|
def request_seen(self, request):
|
|
return False
|
|
|
|
def open(self): # can return deferred
|
|
pass
|
|
|
|
def close(self, reason): # can return a deferred
|
|
pass
|
|
|
|
def log(self, request, spider): # log that a request has been filtered
|
|
pass
|
|
|
|
|
|
class RFPDupeFilter(BaseDupeFilter):
|
|
"""Request Fingerprint duplicates filter"""
|
|
|
|
def __init__(self, path=None, debug=False):
|
|
self.file = None
|
|
self.fingerprints = set()
|
|
self.logdupes = True
|
|
self.debug = debug
|
|
if path:
|
|
self.file = open(os.path.join(path, 'requests.seen'), 'a+')
|
|
self.fingerprints.update(x.rstrip() for x in self.file)
|
|
|
|
@classmethod
|
|
def from_settings(cls, settings):
|
|
debug = settings.getbool('DUPEFILTER_DEBUG')
|
|
return cls(job_dir(settings), debug)
|
|
|
|
def request_seen(self, request):
|
|
fp = self.request_fingerprint(request)
|
|
if fp in self.fingerprints:
|
|
return True
|
|
self.fingerprints.add(fp)
|
|
if self.file:
|
|
self.file.write(fp + os.linesep)
|
|
|
|
def request_fingerprint(self, request):
|
|
return request_fingerprint(request)
|
|
|
|
def close(self, reason):
|
|
if self.file:
|
|
self.file.close()
|
|
|
|
def log(self, request, spider):
|
|
if self.debug:
|
|
fmt = "Filtered duplicate request: %(request)s"
|
|
log.msg(format=fmt, request=request, level=log.DEBUG, spider=spider)
|
|
elif self.logdupes:
|
|
fmt = ("Filtered duplicate request: %(request)s"
|
|
" - no more duplicates will be shown"
|
|
" (see DUPEFILTER_DEBUG to show all duplicates)")
|
|
log.msg(format=fmt, request=request, level=log.DEBUG, spider=spider)
|
|
self.logdupes = False
|
|
|
|
spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
|