From 41a9e3be32ed7564295cde253095f964c0bade55 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 15 Jul 2019 00:12:31 +0500 Subject: [PATCH] ScrapyPriorityQueue works with disk --- scrapy/core/scheduler.py | 3 ++- scrapy/pqueues.py | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 1aebed2c3..7676e610a 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -164,7 +164,8 @@ class Scheduler(object): q = create_instance(self.pqclass, None, self.crawler, - self._newdq, + self.dqclass, + self.dqdir, state) if q: logger.info("Resuming crawl (%(queuesize)d requests scheduled)", diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 9bebe3b7b..d8f325042 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -118,9 +118,10 @@ class ScrapyPriorityQueue(object): slot_startprios.__class__) for priority, state in startprios.items(): + priority = int(priority) self.queues[priority] = self.qfactory(priority, state) - self.curprio = min(startprios) + self.curprio = min(int(p) for p in startprios) def qfactory(self, key, startprios=()): return self.downstream_queue_cls(self.crawler, @@ -150,9 +151,7 @@ class ScrapyPriorityQueue(object): def close(self): active = {} for p, q in self.queues.items(): - if len(q): - active.append(p) - q.close() + active[p] = q.close() return active def __len__(self):