more simple downstream queue interface

This commit is contained in:
Vostretsov Nikita 2019-07-15 10:34:14 +05:00
parent d79f49c4b3
commit 231eec8646
2 changed files with 4 additions and 6 deletions

View File

@ -52,11 +52,9 @@ class ScrapyPriorityQueue(object):
self.curprio = min(p for p in startprios)
def qfactory(self, key, startprios=()):
def qfactory(self, key):
return self.downstream_queue_cls(self.crawler,
None,
self.key + '/' + str(key),
startprios)
self.key + '/' + str(key))
def push(self, request):
priority = -request.priority

View File

@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class):
class ScrapyRequestQueue(queue_class):
def __init__(self, crawler, downstream_queue_cls, key, startprios):
def __init__(self, crawler, key):
self.spider = crawler.spider
super(ScrapyRequestQueue, self).__init__(key)
@ -70,7 +70,7 @@ def _scrapy_serialization_queue(queue_class):
def _scrapy_in_memory_queue(queue_class):
class ScrapyRequestQueue(queue_class):
def __init__(self, crawler, downstream_queue_cls, key, startprios):
def __init__(self, crawler, key):
super(ScrapyRequestQueue, self).__init__()
return ScrapyRequestQueue