mirror of https://github.com/scrapy/scrapy.git
97 lines
3.6 KiB
Python
97 lines
3.6 KiB
Python
from twisted.internet import defer
|
|
|
|
from scrapy.http import Request
|
|
from scrapy.utils.misc import arg_to_iter
|
|
from scrapy.utils.spider import create_spider_for_request
|
|
from scrapy.utils.python import stringify_dict
|
|
from scrapy import log
|
|
|
|
|
|
class ExecutionQueue(object):
|
|
|
|
def __init__(self, spiders, queue, poll_interval=5, keep_alive=False):
|
|
self.spider_requests = []
|
|
self.poll_interval = poll_interval
|
|
self._spiders = spiders
|
|
self._queue = queue
|
|
self._keepalive = keep_alive
|
|
|
|
@defer.inlineCallbacks
|
|
def _append_next(self):
|
|
"""Called when there are no more items left in self.spider_requests.
|
|
This method is meant to be overriden in subclasses to add new (spider,
|
|
requests) tuples to self.spider_requests. It can return a Deferred.
|
|
"""
|
|
msg = yield self._queue.pop()
|
|
if msg:
|
|
name = msg.pop('name')
|
|
msg = stringify_dict(msg) # see #250
|
|
self.append_spider_name(name, **msg)
|
|
|
|
def get_next(self):
|
|
"""Return a tuple (spider, requests) containing a list of Requests and
|
|
the Spider which will be used to crawl those Requests. If there aren't
|
|
any more spiders to crawl it must return (None, []).
|
|
|
|
This method can return a deferred.
|
|
"""
|
|
if self.spider_requests:
|
|
return self._get_next_now()
|
|
d = defer.maybeDeferred(self._append_next)
|
|
d.addCallback(lambda _: self._get_next_now())
|
|
return d
|
|
|
|
def _get_next_now(self):
|
|
try:
|
|
return self.spider_requests.pop(0)
|
|
except IndexError:
|
|
return (None, [])
|
|
|
|
def is_finished(self):
|
|
"""Return True if the queue is empty and there won't be any more
|
|
spiders to crawl (this is for one-shot runs). If it returns ``False``
|
|
Scrapy will keep polling this queue for new requests to scrape
|
|
"""
|
|
return not self._keepalive and not bool(self.spider_requests)
|
|
|
|
def append_spider(self, spider):
|
|
"""Append a Spider to crawl"""
|
|
requests = spider.start_requests()
|
|
self.spider_requests.append((spider, requests))
|
|
|
|
def append_request(self, request, spider=None, **kwargs):
|
|
if spider is None:
|
|
spider = create_spider_for_request(self._spiders, request, **kwargs)
|
|
if spider:
|
|
self.spider_requests.append((spider, [request]))
|
|
|
|
def append_url(self, url=None, spider=None, **kwargs):
|
|
"""Append a URL to crawl with the given spider. If the spider is not
|
|
given, a spider will be looked up based on the URL
|
|
"""
|
|
if url is None:
|
|
raise ValueError("A url is required")
|
|
if spider is None:
|
|
spider = create_spider_for_request(self._spiders, Request(url), \
|
|
**kwargs)
|
|
if spider:
|
|
requests = arg_to_iter(spider.make_requests_from_url(url))
|
|
self.spider_requests.append((spider, requests))
|
|
|
|
def append_spider_name(self, name=None, **spider_kwargs):
|
|
"""Append a spider to crawl given its name and optional arguments,
|
|
which are used to instantiate it. The SpiderManager is used to lookup
|
|
the spider
|
|
"""
|
|
if name is None:
|
|
raise ValueError("A spider name is required")
|
|
try:
|
|
spider = self._spiders.create(name, **spider_kwargs)
|
|
except KeyError:
|
|
log.msg('Unable to find spider: %s' % name, log.ERROR)
|
|
except:
|
|
log.err(None, 'Error creating spider %r with arguments: %r' % \
|
|
(name, spider_kwargs))
|
|
else:
|
|
self.append_spider(spider)
|