RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Logging the request referer when DUPEFILTER_DEBUG is active

This commit is contained in:
Júlio César Batista 2019-01-18 11:38:59 -02:00
parent 094dde6fdb
commit bdf12f7750
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/dupefilters.py
View File

@ -3,8 +3,7 @@ import os
import logging
from scrapy.utils.job import job_dir
from scrapy.utils.request import request_fingerprint
from scrapy.utils.request import referer_str, request_fingerprint
class BaseDupeFilter(object):
@ -61,8 +60,9 @@ class RFPDupeFilter(BaseDupeFilter):
def log(self, request, spider):
if self.debug:
msg = "Filtered duplicate request: %(request)s"
self.logger.debug(msg, {'request': request}, extra={'spider': spider})
msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
args = {'request': request, 'referer': referer_str(request) }
self.logger.debug(msg, args, extra={'spider': spider})
elif self.logdupes:
msg = ("Filtered duplicate request: %(request)s"
" - no more duplicates will be shown"