mirror of https://github.com/scrapy/scrapy.git
More core changes:
* removed execution queue (replaced by newer spider queues) * added real support for returning iterators in Spider.start_requests() * removed support for passing urls to 'scrapy crawl' command
This commit is contained in:
Pablo Hoffman
parent
4dadeb7ccb
commit
84f518fc5e
|
|
@ -218,22 +218,16 @@ Usage example::
|
|||
crawl
|
||||
-----
|
||||
|
||||
* Syntax: ``scrapy crawl <spider|url>``
|
||||
* Syntax: ``scrapy crawl <spider>``
|
||||
* Requires project: *yes*
|
||||
|
||||
Start crawling a spider. If a URL is passed instead of a spider, it will start
|
||||
from that URL instead of the spider start urls.
|
||||
Start crawling a spider.
|
||||
|
||||
Usage examples::
|
||||
|
||||
$ scrapy crawl example.com
|
||||
[ ... example.com spider starts crawling ... ]
|
||||
|
||||
$ scrapy crawl myspider
|
||||
[ ... myspider starts crawling ... ]
|
||||
|
||||
$ scrapy crawl http://example.com/some/page.html
|
||||
[ ... spider that handles example.com starts crawling from that url ... ]
|
||||
|
||||
.. command:: server
|
||||
|
||||
|
|
|
|||
|
|
@ -1,32 +1,21 @@
|
|||
from w3lib.url import is_url
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.conf import settings
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
from scrapy.exceptions import UsageError
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = True
|
||||
|
||||
def syntax(self):
|
||||
return "[options] <spider|url> ..."
|
||||
return "[options] <spider>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Start crawling from a spider or URL"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None, \
|
||||
help="always use this spider when arguments are urls")
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \
|
||||
help="don't follow links (for use with URLs only)")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
|
|
@ -34,49 +23,11 @@ class Command(ScrapyCommand):
|
|||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
if opts.nofollow:
|
||||
settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
|
||||
|
||||
def run(self, args, opts):
|
||||
q = self.crawler.queue
|
||||
urls, names = self._split_urls_and_names(args)
|
||||
for name in names:
|
||||
q.append_spider_name(name, **opts.spargs)
|
||||
|
||||
if opts.spider:
|
||||
try:
|
||||
spider = self.crawler.spiders.create(opts.spider, **opts.spargs)
|
||||
for url in urls:
|
||||
q.append_url(url, spider)
|
||||
except KeyError:
|
||||
log.msg('Unable to find spider: %s' % opts.spider, log.ERROR)
|
||||
else:
|
||||
for name, urls in self._group_urls_by_spider(urls):
|
||||
spider = self.crawler.spiders.create(name, **opts.spargs)
|
||||
for url in urls:
|
||||
q.append_url(url, spider)
|
||||
if len(args) < 1:
|
||||
raise UsageError()
|
||||
for spname in args:
|
||||
spider = self.crawler.spiders.create(spname, **opts.spargs)
|
||||
self.crawler.crawl(spider)
|
||||
self.crawler.start()
|
||||
|
||||
def _group_urls_by_spider(self, urls):
|
||||
spider_urls = defaultdict(list)
|
||||
for url in urls:
|
||||
spider_names = self.crawler.spiders.find_by_request(Request(url))
|
||||
if not spider_names:
|
||||
log.msg('Could not find spider that handles url: %s' % url,
|
||||
log.ERROR)
|
||||
elif len(spider_names) > 1:
|
||||
log.msg('More than one spider can handle url: %s - %s' % \
|
||||
(url, ", ".join(spider_names)), log.ERROR)
|
||||
else:
|
||||
spider_urls[spider_names[0]].append(url)
|
||||
return spider_urls.items()
|
||||
|
||||
def _split_urls_and_names(self, args):
|
||||
urls = []
|
||||
names = []
|
||||
for arg in args:
|
||||
if is_url(arg):
|
||||
urls.append(arg)
|
||||
else:
|
||||
names.append(arg)
|
||||
return urls, names
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from w3lib.url import is_url
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.http import Request
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.exceptions import UsageError
|
||||
from scrapy.utils.spider import create_spider_for_request
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
|
|
@ -49,12 +49,10 @@ class Command(ScrapyCommand):
|
|||
|
||||
spider = None
|
||||
if opts.spider:
|
||||
try:
|
||||
spider = self.crawler.spiders.create(opts.spider)
|
||||
except KeyError:
|
||||
log.msg("Could not find spider: %s" % opts.spider, log.ERROR)
|
||||
|
||||
self.crawler.queue.append_request(request, spider, \
|
||||
default_spider=BaseSpider('default'))
|
||||
spider = self.crawler.spiders.create(opts.spider)
|
||||
else:
|
||||
spider = create_spider_for_request(self.crawler.spiders, request, \
|
||||
default_spider=BaseSpider('default'))
|
||||
self.crawler.crawl(spider, [request])
|
||||
self.crawler.start()
|
||||
|
||||
|
|
|
|||
|
|
@ -84,7 +84,7 @@ class Command(ScrapyCommand):
|
|||
spider = self.get_spider(request, opts)
|
||||
if not spider:
|
||||
return None, None
|
||||
self.crawler.queue.append_request(request, spider)
|
||||
self.crawler.crawl(spider, [request])
|
||||
self.crawler.start()
|
||||
if not responses:
|
||||
log.msg('No response downloaded for: %s' % request, log.ERROR, \
|
||||
|
|
|
|||
|
|
@ -60,6 +60,6 @@ class Command(ScrapyCommand):
|
|||
if not spclasses:
|
||||
raise UsageError("No spider found in file: %s\n" % filename)
|
||||
spider = spclasses.pop()(**opts.spargs)
|
||||
# schedule spider and start engine
|
||||
self.crawler.queue.append_spider(spider)
|
||||
|
||||
self.crawler.crawl(spider)
|
||||
self.crawler.start()
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from scrapy import log
|
|||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
default_settings = {'KEEP_ALIVE': True}
|
||||
default_settings = {'KEEP_ALIVE': True, 'LOGSTATS_INTERVAL': 0}
|
||||
|
||||
def syntax(self):
|
||||
return "[url|file]"
|
||||
|
|
|
|||
|
|
@ -21,9 +21,11 @@ from scrapy.utils.defer import mustbe_deferred
|
|||
|
||||
class Slot(object):
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, start_requests, close_if_idle):
|
||||
self.closing = False
|
||||
self.inprogress = set() # requests in progress
|
||||
self.requests = iter(start_requests)
|
||||
self.close_if_idle = close_if_idle
|
||||
|
||||
def add_request(self, request):
|
||||
self.inprogress.add(request)
|
||||
|
|
@ -107,7 +109,13 @@ class ExecutionEngine(object):
|
|||
break
|
||||
|
||||
if self.spider_is_idle(spider):
|
||||
self._spider_idle(spider)
|
||||
slot = self.slots[spider]
|
||||
try:
|
||||
request = slot.requests.next()
|
||||
self.crawl(request, spider)
|
||||
except StopIteration:
|
||||
if slot.close_if_idle:
|
||||
self._spider_idle(spider)
|
||||
|
||||
def _needs_backout(self, spider):
|
||||
slot = self.slots[spider]
|
||||
|
|
@ -212,11 +220,11 @@ class ExecutionEngine(object):
|
|||
return dwld
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def open_spider(self, spider):
|
||||
def open_spider(self, spider, start_requests=None, close_if_idle=True):
|
||||
assert self.has_capacity(), "No free spider slots when opening %r" % \
|
||||
spider.name
|
||||
log.msg("Spider opened", spider=spider)
|
||||
self.slots[spider] = Slot()
|
||||
self.slots[spider] = Slot(start_requests or (), close_if_idle)
|
||||
yield self.scheduler.open_spider(spider)
|
||||
self.downloader.open_spider(spider)
|
||||
yield self.scraper.open_spider(spider)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import signal
|
|||
from twisted.internet import reactor, defer
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy.queue import ExecutionQueue
|
||||
from scrapy.core.engine import ExecutionEngine
|
||||
from scrapy.extension import ExtensionManager
|
||||
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
|
||||
|
|
@ -34,51 +33,25 @@ class Crawler(object):
|
|||
self.extensions = ExtensionManager.from_settings(self.settings)
|
||||
spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
|
||||
self.spiders = spman_cls.from_settings(self.settings)
|
||||
spq_cls = load_object(self.settings['SPIDER_QUEUE_CLASS'])
|
||||
spq = spq_cls.from_settings(self.settings)
|
||||
keepalive = self.settings.getbool('KEEP_ALIVE')
|
||||
pollint = self.settings.getfloat('QUEUE_POLL_INTERVAL')
|
||||
self.queue = ExecutionQueue(self.spiders, spq, poll_interval=pollint,
|
||||
keep_alive=keepalive)
|
||||
self.engine = ExecutionEngine(self.settings, self._spider_closed)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start_next_spider(self):
|
||||
spider, requests = yield defer.maybeDeferred(self.queue.get_next)
|
||||
if spider:
|
||||
self._start_spider(spider, requests)
|
||||
if self.engine.has_capacity() and not self._nextcall.active():
|
||||
self._nextcall = reactor.callLater(self.queue.poll_interval, \
|
||||
self._spider_closed)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _start_spider(self, spider, requests):
|
||||
"""Don't call this method. Use self.queue to start new spiders"""
|
||||
def crawl(self, spider, requests=None):
|
||||
spider.set_crawler(self)
|
||||
yield defer.maybeDeferred(self.engine.open_spider, spider)
|
||||
for request in requests:
|
||||
self.engine.crawl(request, spider)
|
||||
if requests is None:
|
||||
requests = spider.start_requests()
|
||||
return self.engine.open_spider(spider, requests)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _spider_closed(self, spider=None):
|
||||
if not self.engine.open_spiders:
|
||||
is_finished = yield defer.maybeDeferred(self.queue.is_finished)
|
||||
if is_finished:
|
||||
self.stop()
|
||||
return
|
||||
if self.engine.has_capacity():
|
||||
self._start_next_spider()
|
||||
self.stop()
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def start(self):
|
||||
yield defer.maybeDeferred(self.configure)
|
||||
yield defer.maybeDeferred(self.engine.start)
|
||||
self._nextcall = reactor.callLater(0, self._start_next_spider)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def stop(self):
|
||||
if self._nextcall.active():
|
||||
self._nextcall.cancel()
|
||||
if self.engine.running:
|
||||
yield defer.maybeDeferred(self.engine.stop)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,96 +0,0 @@
|
|||
from twisted.internet import defer
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.spider import create_spider_for_request
|
||||
from scrapy.utils.python import stringify_dict
|
||||
from scrapy import log
|
||||
|
||||
|
||||
class ExecutionQueue(object):
|
||||
|
||||
def __init__(self, spiders, queue, poll_interval=5, keep_alive=False):
|
||||
self.spider_requests = []
|
||||
self.poll_interval = poll_interval
|
||||
self._spiders = spiders
|
||||
self._queue = queue
|
||||
self._keepalive = keep_alive
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _append_next(self):
|
||||
"""Called when there are no more items left in self.spider_requests.
|
||||
This method is meant to be overriden in subclasses to add new (spider,
|
||||
requests) tuples to self.spider_requests. It can return a Deferred.
|
||||
"""
|
||||
msg = yield self._queue.pop()
|
||||
if msg:
|
||||
name = msg.pop('name')
|
||||
msg = stringify_dict(msg) # see #250
|
||||
self.append_spider_name(name, **msg)
|
||||
|
||||
def get_next(self):
|
||||
"""Return a tuple (spider, requests) containing a list of Requests and
|
||||
the Spider which will be used to crawl those Requests. If there aren't
|
||||
any more spiders to crawl it must return (None, []).
|
||||
|
||||
This method can return a deferred.
|
||||
"""
|
||||
if self.spider_requests:
|
||||
return self._get_next_now()
|
||||
d = defer.maybeDeferred(self._append_next)
|
||||
d.addCallback(lambda _: self._get_next_now())
|
||||
return d
|
||||
|
||||
def _get_next_now(self):
|
||||
try:
|
||||
return self.spider_requests.pop(0)
|
||||
except IndexError:
|
||||
return (None, [])
|
||||
|
||||
def is_finished(self):
|
||||
"""Return True if the queue is empty and there won't be any more
|
||||
spiders to crawl (this is for one-shot runs). If it returns ``False``
|
||||
Scrapy will keep polling this queue for new requests to scrape
|
||||
"""
|
||||
return not self._keepalive and not bool(self.spider_requests)
|
||||
|
||||
def append_spider(self, spider):
|
||||
"""Append a Spider to crawl"""
|
||||
requests = spider.start_requests()
|
||||
self.spider_requests.append((spider, requests))
|
||||
|
||||
def append_request(self, request, spider=None, **kwargs):
|
||||
if spider is None:
|
||||
spider = create_spider_for_request(self._spiders, request, **kwargs)
|
||||
if spider:
|
||||
self.spider_requests.append((spider, [request]))
|
||||
|
||||
def append_url(self, url=None, spider=None, **kwargs):
|
||||
"""Append a URL to crawl with the given spider. If the spider is not
|
||||
given, a spider will be looked up based on the URL
|
||||
"""
|
||||
if url is None:
|
||||
raise ValueError("A url is required")
|
||||
if spider is None:
|
||||
spider = create_spider_for_request(self._spiders, Request(url), \
|
||||
**kwargs)
|
||||
if spider:
|
||||
requests = arg_to_iter(spider.make_requests_from_url(url))
|
||||
self.spider_requests.append((spider, requests))
|
||||
|
||||
def append_spider_name(self, name=None, **spider_kwargs):
|
||||
"""Append a spider to crawl given its name and optional arguments,
|
||||
which are used to instantiate it. The SpiderManager is used to lookup
|
||||
the spider
|
||||
"""
|
||||
if name is None:
|
||||
raise ValueError("A spider name is required")
|
||||
try:
|
||||
spider = self._spiders.create(name, **spider_kwargs)
|
||||
except KeyError:
|
||||
log.msg('Unable to find spider: %s' % name, log.ERROR)
|
||||
except:
|
||||
log.err(None, 'Error creating spider %r with arguments: %r' % \
|
||||
(name, spider_kwargs))
|
||||
else:
|
||||
self.append_spider(spider)
|
||||
|
|
@ -174,8 +174,6 @@ ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
|
|||
# Item pipelines are typically set in specific commands settings
|
||||
ITEM_PIPELINES = []
|
||||
|
||||
KEEP_ALIVE = False
|
||||
|
||||
LOG_ENABLED = True
|
||||
LOG_ENCODING = 'utf-8'
|
||||
LOG_FORMATTER = 'scrapy.logformatter.LogFormatter'
|
||||
|
|
@ -203,8 +201,6 @@ MEMUSAGE_WARNING_MB = 0
|
|||
|
||||
NEWSPIDER_MODULE = ''
|
||||
|
||||
QUEUE_POLL_INTERVAL = 5
|
||||
|
||||
RANDOMIZE_DOWNLOAD_DELAY = True
|
||||
|
||||
REDIRECT_ENABLED = True
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ class Shell(object):
|
|||
spider = create_spider_for_request(self.crawler.spiders, request, \
|
||||
BaseSpider('default'), log_multiple=True)
|
||||
spider.set_crawler(self.crawler)
|
||||
self.crawler.engine.open_spider(spider)
|
||||
self.crawler.engine.open_spider(spider, close_if_idle=False)
|
||||
d = request_deferred(request)
|
||||
d.addCallback(lambda x: (x, spider))
|
||||
self.crawler.engine.crawl(request, spider)
|
||||
|
|
|
|||
|
|
@ -103,9 +103,6 @@ class GenspiderCommandTest(CommandTest):
|
|||
|
||||
class MiscCommandsTest(CommandTest):
|
||||
|
||||
def test_crawl(self):
|
||||
self.assertEqual(0, self.call('crawl'))
|
||||
|
||||
def test_list(self):
|
||||
self.assertEqual(0, self.call('list'))
|
||||
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ class CrawlerRun(object):
|
|||
self.crawler = get_crawler()
|
||||
self.crawler.install()
|
||||
self.crawler.configure()
|
||||
self.crawler.queue.append_spider(self.spider)
|
||||
self.crawler.crawl(self.spider)
|
||||
self.crawler.start()
|
||||
|
||||
self.deferred = defer.Deferred()
|
||||
|
|
|
|||
|
|
@ -1,130 +0,0 @@
|
|||
import unittest
|
||||
|
||||
from scrapy.queue import ExecutionQueue
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.http import Request
|
||||
|
||||
class TestSpider(BaseSpider):
|
||||
|
||||
name = "default"
|
||||
|
||||
def start_requests(self):
|
||||
return [Request("http://www.example.com/1"), \
|
||||
Request("http://www.example.com/2")]
|
||||
|
||||
def make_requests_from_url(self, url):
|
||||
return [Request(url + "/make1"), Request(url + "/make2")]
|
||||
|
||||
|
||||
class TestSpiderManager(object):
|
||||
|
||||
def find_by_request(self, request):
|
||||
return ['create_for_request']
|
||||
|
||||
def create(self, spider_name, **spider_kwargs):
|
||||
return TestSpider(spider_name, **spider_kwargs)
|
||||
|
||||
|
||||
class ExecutionQueueTest(unittest.TestCase):
|
||||
|
||||
keep_alive = False
|
||||
|
||||
def setUp(self):
|
||||
self.queue = ExecutionQueue(TestSpiderManager(), None, keep_alive=self.keep_alive)
|
||||
self.spider = TestSpider()
|
||||
self.request = Request('about:none')
|
||||
|
||||
def tearDown(self):
|
||||
del self.queue, self.spider, self.request
|
||||
|
||||
def test_is_finished(self):
|
||||
self.assert_(self.queue.is_finished())
|
||||
self.queue.append_request(self.request, self.spider)
|
||||
self.assert_(not self.queue.is_finished())
|
||||
|
||||
def test_append_spider(self):
|
||||
spider = TestSpider()
|
||||
self.queue.append_spider(spider)
|
||||
self.assert_(self.queue.spider_requests[0][0] is spider)
|
||||
self._assert_request_urls(self.queue.spider_requests[0][1],
|
||||
["http://www.example.com/1", "http://www.example.com/2"])
|
||||
|
||||
def test_append_request1(self):
|
||||
spider = TestSpider()
|
||||
request = Request('about:blank')
|
||||
self.queue.append_request(request, spider=spider)
|
||||
self.assert_(self.queue.spider_requests[0][0] is spider)
|
||||
self.assert_(self.queue.spider_requests[0][1][0] is request)
|
||||
|
||||
def test_append_request2(self):
|
||||
request = Request('about:blank')
|
||||
self.queue.append_request(request, arg='123')
|
||||
spider = self.queue.spider_requests[0][0]
|
||||
self.assert_(spider.name == 'create_for_request')
|
||||
self.assert_(spider.arg == '123')
|
||||
|
||||
def test_append_url(self):
|
||||
spider = TestSpider()
|
||||
url = 'http://www.example.com/asd'
|
||||
self.queue.append_url(url, spider=spider)
|
||||
self.assert_(self.queue.spider_requests[0][0] is spider)
|
||||
self._assert_request_urls(self.queue.spider_requests[0][1], \
|
||||
['http://www.example.com/asd/make1', 'http://www.example.com/asd/make2'])
|
||||
|
||||
def test_append_url_kwarg(self):
|
||||
spider = TestSpider()
|
||||
url = 'http://www.example.com/asd'
|
||||
self.queue.append_url(url=url, spider=spider)
|
||||
self.assert_(self.queue.spider_requests[0][0] is spider)
|
||||
self._assert_request_urls(self.queue.spider_requests[0][1], \
|
||||
['http://www.example.com/asd/make1', 'http://www.example.com/asd/make2'])
|
||||
|
||||
def test_append_url2(self):
|
||||
url = 'http://www.example.com/asd'
|
||||
self.queue.append_url(url, arg='123')
|
||||
self._assert_request_urls(self.queue.spider_requests[0][1], \
|
||||
['http://www.example.com/asd/make1', 'http://www.example.com/asd/make2'])
|
||||
spider = self.queue.spider_requests[0][0]
|
||||
self.assert_(spider.name == 'create_for_request')
|
||||
self.assert_(spider.arg == '123')
|
||||
|
||||
def test_append_spider_name(self):
|
||||
self.queue.append_spider_name('test123', arg='123')
|
||||
spider = self.queue.spider_requests[0][0]
|
||||
self.assert_(spider.name == 'test123')
|
||||
self.assert_(spider.arg == '123')
|
||||
|
||||
def test_append_spider_name_kwarg(self):
|
||||
self.queue.append_spider_name(name='test123', arg='123')
|
||||
spider = self.queue.spider_requests[0][0]
|
||||
self.assert_(spider.name == 'test123')
|
||||
self.assert_(spider.arg == '123')
|
||||
|
||||
def test_append_next(self):
|
||||
# the reason for this test: http://dev.scrapy.org/ticket/250
|
||||
class MockQueue(object):
|
||||
def pop(self):
|
||||
return {u'name': u'test123', u'test': u'hello'}
|
||||
self.queue._queue = MockQueue()
|
||||
self.queue._append_next()
|
||||
spider = self.queue.spider_requests[0][0]
|
||||
self.assert_(spider.name == 'test123')
|
||||
self.assert_(spider.test == 'hello')
|
||||
|
||||
def _assert_request_urls(self, requests, urls):
|
||||
assert all(isinstance(x, Request) for x in requests)
|
||||
self.assertEqual([x.url for x in requests], urls)
|
||||
|
||||
class KeepAliveExecutionQueueTest(ExecutionQueueTest):
|
||||
|
||||
keep_alive = True
|
||||
|
||||
def test_is_finished(self):
|
||||
self.assert_(not self.queue.is_finished())
|
||||
self.queue.append_request(self.request, self.spider)
|
||||
self.assert_(not self.queue.is_finished())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Loading…
Reference in New Issue