RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3588 from ejulio/dupefilter-debug 

[MRG+1] Adding requests referer to RFPDupeFilter log messages
This commit is contained in:
Daniel Graña 2019-01-28 10:22:48 -03:00 committed by GitHub
parent 8fca98616a 8eade7d864
commit c4f57608d6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 60 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/dupefilters.py
View File

@ -3,8 +3,7 @@ import os
import logging
from scrapy.utils.job import job_dir
from scrapy.utils.request import request_fingerprint
from scrapy.utils.request import referer_str, request_fingerprint
class BaseDupeFilter(object):
@ -61,8 +60,9 @@ class RFPDupeFilter(BaseDupeFilter):
def log(self, request, spider):
if self.debug:
msg = "Filtered duplicate request: %(request)s"
self.logger.debug(msg, {'request': request}, extra={'spider': spider})
msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
args = {'request': request, 'referer': referer_str(request) }
self.logger.debug(msg, args, extra={'spider': spider})
elif self.logdupes:
msg = ("Filtered duplicate request: %(request)s"
" - no more duplicates will be shown"

57
tests/test_dupefilters.py
View File

@ -2,6 +2,7 @@ import hashlib
import tempfile
import unittest
import shutil
from testfixtures import LogCapture
from scrapy.dupefilters import RFPDupeFilter
from scrapy.http import Request
@ -9,7 +10,7 @@ from scrapy.core.scheduler import Scheduler
from scrapy.utils.python import to_bytes
from scrapy.utils.job import job_dir
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
class FromCrawlerRFPDupeFilter(RFPDupeFilter):
@ -126,3 +127,57 @@ class RFPDupeFilterTest(unittest.TestCase):
assert case_insensitive_dupefilter.request_seen(r2)
case_insensitive_dupefilter.close('finished')
def test_log(self):
with LogCapture() as l:
settings = {'DUPEFILTER_DEBUG': False,
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
spider = SimpleSpider.from_crawler(crawler)
dupefilter = scheduler.df
dupefilter.open()
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html')
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' - no more duplicates will be shown'
' (see DUPEFILTER_DEBUG to show all duplicates)')))
dupefilter.close('finished')
def test_log_debug(self):
with LogCapture() as l:
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
spider = SimpleSpider.from_crawler(crawler)
dupefilter = scheduler.df
dupefilter.open()
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html',
headers={'Referer': 'http://scrapytest.org/INDEX.html'}
)
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' (referer: None)')))
l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' (referer: http://scrapytest.org/INDEX.html)')))
dupefilter.close('finished')