mirror of https://github.com/scrapy/scrapy.git
more simple downstream queue interface
This commit is contained in:
parent
d79f49c4b3
commit
231eec8646
|
|
@ -52,11 +52,9 @@ class ScrapyPriorityQueue(object):
|
|||
|
||||
self.curprio = min(p for p in startprios)
|
||||
|
||||
def qfactory(self, key, startprios=()):
|
||||
def qfactory(self, key):
|
||||
return self.downstream_queue_cls(self.crawler,
|
||||
None,
|
||||
self.key + '/' + str(key),
|
||||
startprios)
|
||||
self.key + '/' + str(key))
|
||||
|
||||
def push(self, request):
|
||||
priority = -request.priority
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class):
|
|||
|
||||
class ScrapyRequestQueue(queue_class):
|
||||
|
||||
def __init__(self, crawler, downstream_queue_cls, key, startprios):
|
||||
def __init__(self, crawler, key):
|
||||
self.spider = crawler.spider
|
||||
super(ScrapyRequestQueue, self).__init__(key)
|
||||
|
||||
|
|
@ -70,7 +70,7 @@ def _scrapy_serialization_queue(queue_class):
|
|||
def _scrapy_in_memory_queue(queue_class):
|
||||
|
||||
class ScrapyRequestQueue(queue_class):
|
||||
def __init__(self, crawler, downstream_queue_cls, key, startprios):
|
||||
def __init__(self, crawler, key):
|
||||
super(ScrapyRequestQueue, self).__init__()
|
||||
|
||||
return ScrapyRequestQueue
|
||||
|
|
|
|||
Loading…
Reference in New Issue