RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4385 from nyov/obsolete 

Obsolete REDIRECT_MAX_METAREFRESH_DELAY, LOG_UNSERIALIZABLE_REQUESTS
This commit is contained in:
Mikhail Korobov 2020-03-05 00:04:29 +05:00 committed by GitHub
parent 29f957d8cb 1b591ff061
commit ff8e826dd6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 3 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapy/core/scheduler.py
View File

@ -66,8 +66,7 @@ class Scheduler(object):
dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS',
settings.getbool('SCHEDULER_DEBUG'))
logunser = settings.getbool('SCHEDULER_DEBUG')
return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
mqclass=mqclass, crawler=crawler)

3
scrapy/downloadermiddlewares/redirect.py
View File

@ -93,8 +93,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
def __init__(self, settings):
super(MetaRefreshMiddleware, self).__init__(settings)
self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY',
settings.getint('METAREFRESH_MAXDELAY'))
self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
def process_response(self, request, response, spider):
if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \

2
scrapy/settings/deprecated.py
View File

@ -11,8 +11,6 @@ DEPRECATED_SETTINGS = [
('SQLITE_DB', 'no longer supported'),
('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'),
('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
('REDIRECT_MAX_METAREFRESH_DELAY', 'use METAREFRESH_MAXDELAY instead'),
('LOG_UNSERIALIZABLE_REQUESTS', 'use SCHEDULER_DEBUG instead'),
]

2
tests/test_scheduler.py
View File

@ -46,7 +46,7 @@ class MockCrawler(Crawler):
def __init__(self, priority_queue_cls, jobdir):
settings = dict(
LOG_UNSERIALIZABLE_REQUESTS=False,
SCHEDULER_DEBUG=False,
SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,