mirror of https://github.com/scrapy/scrapy.git
67 lines
2.4 KiB
Python
67 lines
2.4 KiB
Python
"""
|
|
Crawler class
|
|
|
|
The Crawler class can be used to crawl pages using the Scrapy crawler from
|
|
outside a Scrapy project, for example, from a standalone script.
|
|
|
|
To use it, instantiate it and call the "crawl" method with one (or more)
|
|
requests. For example:
|
|
|
|
>>> from scrapy.crawler import Crawler
|
|
>>> from scrapy.http import Request
|
|
>>> def parse_response(response):
|
|
... print "Visited: %s" % response.url
|
|
...
|
|
>>> request = Request('http://scrapy.org', callback=parse_response)
|
|
>>> crawler = Crawler()
|
|
>>> crawler.crawl(request)
|
|
Visited: http://scrapy.org
|
|
>>>
|
|
|
|
Request callbacks follow the same API of spiders callback, which means that all
|
|
requests returned from the callbacks will be followed.
|
|
|
|
See examples/scripts/count_and_follow_links.py for a more detailed example.
|
|
|
|
WARNING: The Crawler class currently has a big limitation - it cannot be used
|
|
more than once in the same Python process. This is due to the fact that Twisted
|
|
reactors cannot be restarted. Hopefully, this limitation will be removed in the
|
|
future.
|
|
"""
|
|
|
|
from scrapy.xlib.pydispatch import dispatcher
|
|
from scrapy.core.manager import scrapymanager
|
|
from scrapy.core.engine import scrapyengine
|
|
from scrapy.conf import settings as scrapy_settings
|
|
from scrapy import log
|
|
|
|
class Crawler(object):
|
|
|
|
def __init__(self, enable_log=False, stop_on_error=False, silence_errors=False, \
|
|
settings=None):
|
|
self.stop_on_error = stop_on_error
|
|
self.silence_errors = silence_errors
|
|
# disable offsite middleware (by default) because it prevents free crawling
|
|
if settings is not None:
|
|
settings.overrides.update(settings)
|
|
scrapy_settings.overrides['SPIDER_MIDDLEWARES'] = {
|
|
'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': None}
|
|
scrapy_settings.overrides['LOG_ENABLED'] = enable_log
|
|
scrapymanager.configure()
|
|
dispatcher.connect(self._logmessage_received, signal=log.logmessage_received)
|
|
|
|
def crawl(self, *args):
|
|
scrapymanager.runonce(*args)
|
|
|
|
def stop(self):
|
|
scrapyengine.stop()
|
|
log.log_level = log.SILENT
|
|
scrapyengine.kill()
|
|
|
|
def _logmessage_received(self, message, level):
|
|
if level <= log.ERROR:
|
|
if not self.silence_errors:
|
|
print "Crawler error: %s" % message
|
|
if self.stop_on_error:
|
|
self.stop()
|