diff --git a/conftest.py b/conftest.py index 9f9a5bca7..aa27ddd2b 100644 --- a/conftest.py +++ b/conftest.py @@ -4,7 +4,7 @@ from twisted.python import log from scrapy import optional_features -collect_ignore = ["scrapy/stats.py"] +collect_ignore = ["scrapy/stats.py", "scrapy/project.py"] if 'django' not in optional_features: collect_ignore.append("tests/test_djangoitem/models.py") diff --git a/docs/faq.rst b/docs/faq.rst index 47bfede71..1d6c56d97 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -280,37 +280,6 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. - -I'm getting an error: "cannot import name crawler" --------------------------------------------------- - -This is caused by Scrapy changes due to the singletons removal. The error is -most likely raised by a module (extension, middleware, pipeline or spider) in -your Scrapy project that imports ``crawler`` from ``scrapy.project``. For -example:: - - from scrapy.project import crawler - - class SomeExtension(object): - def __init__(self): - self.crawler = crawler - # ... - -This way to access the crawler object is deprecated, the code should be ported -to use ``from_crawler`` class method, for example:: - - class SomeExtension(object): - - @classmethod - def from_crawler(cls, crawler): - o = cls() - o.crawler = crawler - return o - -Scrapy command line tool has some backwards compatibility in place to support -the old import mechanism (with a deprecation warning), but this mechanism may -not work if you use Scrapy differently (for example, as a library). - .. _user agents: http://en.wikipedia.org/wiki/User_agent .. _LIFO: http://en.wikipedia.org/wiki/LIFO .. _DFO order: http://en.wikipedia.org/wiki/Depth-first_search diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 341340c2a..03a0b4124 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -28,9 +28,10 @@ contains a dictionary of all available extensions and their order similar to how you :ref:`configure the downloader middlewares `. -.. class:: Crawler(settings) +.. class:: Crawler(spidercls, settings) The Crawler object must be instantiated with a + :class:`scrapy.spider.Spider` subclass and a :class:`scrapy.settings.Settings` object. .. attribute:: settings @@ -75,13 +76,6 @@ how you :ref:`configure the downloader middlewares For an introduction on extensions and a list of available extensions on Scrapy see :ref:`topics-extensions`. - .. attribute:: spiders - - The spider manager which takes care of loading and instantiating - spiders. - - Most extensions won't need to access this attribute. - .. attribute:: engine The execution engine, which coordinates the core crawling logic @@ -91,18 +85,67 @@ how you :ref:`configure the downloader middlewares or modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. - .. method:: configure() + .. attribute:: spider - Configure the crawler. + Spider currently being crawled. This is an instance of the spider class + provided while constructing the crawler, and it is created after the + arguments given in the :meth:`crawl` method. - This loads extensions, middlewares and spiders, leaving the crawler - ready to be started. It also configures the execution engine. + .. method:: crawl(\*args, \**kwargs) - .. method:: start() + Starts the crawler by instantiating its spider class with the given + `args` and `kwargs` arguments, while setting the execution engine in + motion. - Start the crawler. This calls :meth:`configure` if it hasn't been called yet. Returns a deferred that is fired when the crawl is finished. +.. class:: CrawlerRunner(settings) + + This is a convenient helper class that creates, configures and runs + crawlers inside an already setup Twisted `reactor`_. + + The CrawlerRunner object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + .. attribute:: crawlers + + Set of :class:`crawlers ` created by the + :meth:`crawl` method. + + .. attribute:: crawl_deferreds + + Set of the `deferreds`_ return by the :meth:`crawl` method. This + collection it's useful for keeping track of current crawling state. + + .. method:: crawl(spidercls, \*args, \**kwargs) + + This method sets up the crawling of the given `spidercls` with the + provided arguments. + + It takes care of loading the spider class while configuring and starting + a crawler for it. + + Returns a deferred that is fired when the crawl is finished. + + :param spidercls: spider class or spider's name inside the project + :type spidercls: :class:`~scrapy.spider.Spider` subclass or str + + :param args: arguments to initializate the spider + :type args: list + + :param kwargs: keyword arguments to initializate the spider + :type kwargs: dict + + .. method:: stop() + + Stops simultaneously all the crawling jobs taking place. + + Returns a deferred that is fired when they all have ended. + .. _topics-api-settings: Settings API @@ -252,8 +295,8 @@ Settings API .. method:: getlist(name, default=None) - Get a setting value as a list. If the setting original type is a list it - will be returned verbatim. If it's a string it will be split by ",". + Get a setting value as a list. If the setting original type is a list, a + copy of it will be returned. If it's a string it will be split by ",". For example, settings populated through environment variables set to ``'one,two'`` will return a list ['one', 'two'] when using this method. @@ -264,6 +307,90 @@ Settings API :param default: the value to return if no setting is found :type default: any + .. method:: getdict(name, default=None) + + Get a setting value as a dictionary. If the setting original type is a + dictionary, a copy of it will be returned. If it's a string it will + evaluated as a json dictionary. + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + + .. method:: copy() + + Make a deep copy of current settings. + + This method returns a new instance of the :class:`Settings` class, + populated with the same values and their priorities. + + Modifications to the new object won't be reflected on the original + settings. + + .. method:: freeze() + + Disable further changes to the current settings. + + After calling this method, the present state of the settings will become + immutable. Trying to change values through the :meth:`~set` method and + its variants won't be possible and will be alerted. + + .. method:: frozencopy() + + Return an immutable copy of the current settings. + + Alias for a :meth:`~freeze` call in the object returned by :meth:`copy` + +.. _topics-api-spidermanager: + +SpiderManager API +================= + +.. module:: scrapy.spidermanager + :synopsis: The spider manager + +.. class:: SpiderManager + + This class is in charge of retrieving and handling the spider classes + defined across the project. + + Custom spider managers can be employed by specifying their path in the + :setting:`SPIDER_MANAGER_CLASS` project setting. They must fully implement + the :class:`scrapy.interfaces.ISpiderManager` interface to guarantee an + errorless execution. + + .. method:: from_settings(settings) + + This class method is used by Scrapy to create an instance of the class. + It's called with the current project settings, and it loads the spiders + found in the modules of the :setting:`SPIDER_MODULES` setting. + + :param settings: project settings + :type settings: :class:`~scrapy.settings.Settings` instance + + .. method:: load(spider_name) + + Get the Spider class with the given name. It'll look into the previously + loaded spiders for a spider class with name `spider_name` and will raise + a KeyError if not found. + + :param spider_name: spider class name + :type spider_name: str + + .. method:: list() + + Get the names of the available spiders in the project. + + .. method:: find_by_request(request) + + List the spiders' names that can handle the given request. Will try to + match the request's url against the domains of the spiders. + + :param request: queried request + :type request: :class:`~scrapy.http.Request` instance + .. _topics-api-signals: Signals API @@ -384,3 +511,4 @@ class (which they all inherit from). .. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html .. _deferred: http://twistedmatrix.com/documents/current/core/howto/defer.html +.. _reactor: http://twistedmatrix.com/documents/current/core/howto/reactor-basics.html diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 1a9e975d8..819884ac2 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -10,7 +10,11 @@ logging`_ but this may change in the future. .. _Twisted logging: http://twistedmatrix.com/projects/core/documentation/howto/logging.html -The logging service must be explicitly started through the :func:`scrapy.log.start` function. +The logging service must be explicitly started through the +:func:`scrapy.log.start` function to catch the top level Scrapy's log messages. +On top of that, each crawler has its own independent log observer +(automatically attached when it's created) that intercepts its spider's log +messages. .. _topics-logging-levels: @@ -55,8 +59,11 @@ scrapy.log module .. function:: start(logfile=None, loglevel=None, logstdout=None) - Start the logging facility. This must be called before actually logging any - messages. Otherwise, messages logged before this call will get lost. + Start the top level Scrapy logger. This must be called before actually + logging any top level messages (those logged using this module's + :func:`~scrapy.log.msg` function instead of the :meth:`Spider.log + ` method). Otherwise, messages logged before this + call will get lost. :param logfile: the file path to use for logging output. If omitted, the :setting:`LOG_FILE` setting will be used. If both are ``None``, the log diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 64b3a2da7..b188ee562 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -19,8 +19,9 @@ Remember that Scrapy is built on top of the Twisted asynchronous networking library, so you need to run it inside the Twisted reactor. Note that you will also have to shutdown the Twisted reactor yourself after the -spider is finished. This can be achieved by connecting a handler to the -``signals.spider_closed`` signal. +spider is finished. This can be achieved by adding callbacks to the deferred +returned by the :meth:`CrawlerRunner.crawl +` method. What follows is a working example of how to do that, using the `testspiders`_ project as example. @@ -28,20 +29,43 @@ project as example. :: from twisted.internet import reactor - from scrapy.crawler import Crawler - from scrapy import log, signals - from testspiders.spiders.followall import FollowAllSpider + from scrapy.crawler import CrawlerRunner from scrapy.utils.project import get_project_settings - spider = FollowAllSpider(domain='scrapinghub.com') - settings = get_project_settings() - crawler = Crawler(settings) - crawler.signals.connect(reactor.stop, signal=signals.spider_closed) - crawler.configure() - crawler.crawl(spider) - crawler.start() - log.start() - reactor.run() # the script will block here until the spider_closed signal was sent + runner = CrawlerRunner(get_project_settings()) + + # 'followall' is the name of one of the spiders of the project. + d = runner.crawl('followall', domain='scrapinghub.com') + d.addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until the crawling is finished + +Running spiders outside projects it's not much different. You have to create a +generic :class:`~scrapy.settings.Settings` object and populate it as needed +(See :ref:`topics-settings-ref` for the available settings), instead of using +the configuration returned by `get_project_settings`. + +Spiders can still be referenced by their name if :setting:`SPIDER_MODULES` is +set with the modules where Scrapy should look for spiders. Otherwise, passing +the spider class as first argument in the :meth:`CrawlerRunner.crawl +` method is enough. + +:: + + from twisted.internet import reactor + from scrapy.spider import Spider + from scrapy.crawler import CrawlerRunner + from scrapy.settings import Settings + + class MySpider(Spider): + # Your spider definition + ... + + settings = Settings({'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'}) + runner = CrawlerRunner(settings) + + d = runner.crawl(MySpider) + d.addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until the crawling is finished .. seealso:: `Twisted Reactor Overview`_. @@ -52,28 +76,42 @@ By default, Scrapy runs a single spider per process when you run ``scrapy crawl``. However, Scrapy supports running multiple spiders per process using the :ref:`internal API `. -Here is an example, using the `testspiders`_ project: +Here is an example that runs multiple spiders simultaneously, using the +`testspiders`_ project: :: - from twisted.internet import reactor - from scrapy.crawler import Crawler - from scrapy import log - from testspiders.spiders.followall import FollowAllSpider + from twisted.internet import reactor, defer + from scrapy.crawler import CrawlerRunner from scrapy.utils.project import get_project_settings - def setup_crawler(domain): - spider = FollowAllSpider(domain=domain) - settings = get_project_settings() - crawler = Crawler(settings) - crawler.configure() - crawler.crawl(spider) - crawler.start() - + runner = CrawlerRunner(get_project_settings()) + dfs = set() for domain in ['scrapinghub.com', 'insophia.com']: - setup_crawler(domain) - log.start() - reactor.run() + d = runner.crawl('followall', domain=domain) + dfs.add(d) + + defer.DeferredList(dfs).addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until all crawling jobs are finished + +Same example but running the spiders sequentially by chaining the deferreds: + +:: + + from twisted.internet import reactor, defer + from scrapy.crawler import CrawlerRunner + from scrapy.utils.project import get_project_settings + + runner = CrawlerRunner(get_project_settings()) + + @defer.inlineCallbacks + def crawl(): + for domain in ['scrapinghub.com', 'insophia.com']: + yield runner.crawl('followall', domain=domain) + reactor.stop() + + crawl() + reactor.run() # the script will block here until the last crawl call is finished .. seealso:: :ref:`run-from-script`. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0838cfc46..e0c432cb3 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -767,6 +767,16 @@ A dict containing the scrapy contracts enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS` instead. For more info see :ref:`topics-contracts`. +.. setting:: SPIDER_MANAGER_CLASS + +SPIDER_MANAGER_CLASS +-------------------- + +Default: ``'scrapy.spidermanager.SpiderManager'`` + +The class that will be used for handling spiders, which must implement the +:ref:`topics-api-spidermanager`. + .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 37268c3ca..5c1cfbd47 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -186,7 +186,7 @@ Here's an example of how you would call it from your spider:: # We want to inspect one specific response. if ".org" in response.url: from scrapy.shell import inspect_response - inspect_response(response) + inspect_response(response, self) # Rest of parsing code. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 73c34e75f..de8f988c0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -133,6 +133,44 @@ Spider listed here. The subsequent URLs will be generated successively from data contained in the start URLs. + .. attribute:: crawler + + This attribute is set by the :meth:`from_crawler` class method after + initializating the class, and links to the + :class:`~scrapy.crawler.Crawler` object to which this spider instance is + bound. + + Crawlers encapsulate a lot of components in the project for their single + entry access (such as extensions, middlewares, signals managers, etc). + See :ref:`topics-api-crawler` to know more about them. + + .. attribute:: settings + + Configuration on which this spider is been ran. This is a + :class:`~scrapy.settings.Settings` instance, see the + :ref:`topics-settings` topic for a detailed introduction on this subject. + + .. method:: from_crawler(crawler, \*args, \**kwargs) + + This is the class method used by Scrapy to create your spiders. + + You probably won't need to override this directly, since the default + implementation acts as a proxy to the :meth:`__init__` method, calling + it with the given arguments `args` and named arguments `kwargs`. + + Nonetheless, this method sets the :attr:`crawler` and :attr:`settings` + attributes in the new instance, so they can be accessed later inside the + spider's code. + + :param crawler: crawler to which the spider will be bound + :type crawler: :class:`~scrapy.crawler.Crawler` instance + + :param args: arguments passed to the :meth:`__init__` method + :type args: list + + :param kwargs: keyword arguments passed to the :meth:`__init__` method + :type kwargs: dict + .. method:: start_requests() This method must return an iterable with the first Requests to crawl for diff --git a/scrapy/command.py b/scrapy/command.py index b2eb9cf8f..9ac013098 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -2,12 +2,11 @@ Base class for Scrapy commands """ import os -import warnings from optparse import OptionGroup from twisted.python import failure from scrapy.utils.conf import arglist_to_dict -from scrapy.exceptions import UsageError, ScrapyDeprecationWarning +from scrapy.exceptions import UsageError class ScrapyCommand(object): @@ -27,31 +26,6 @@ class ScrapyCommand(object): assert not hasattr(self, '_crawler'), "crawler already set" self._crawler = crawler - @property - def crawler(self): - warnings.warn("Command's default `crawler` is deprecated and will be removed. " - "Use `create_crawler` method to instatiate crawlers.", - ScrapyDeprecationWarning) - - if not hasattr(self, '_crawler'): - crawler = self.crawler_process.create_crawler() - - old_start = crawler.start - self.crawler_process.started = False - - def wrapped_start(): - if self.crawler_process.started: - old_start() - else: - self.crawler_process.started = True - self.crawler_process.start() - - crawler.start = wrapped_start - - self.set_crawler(crawler) - - return self._crawler - def syntax(self): """ Command syntax (preferably one-line). Do not include command name. diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 18934f976..395597546 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -22,9 +22,7 @@ class Command(ScrapyCommand): def run(self, args, opts): with _BenchServer(): - spider = _BenchSpider(total=100000) - crawler = self.crawler_process.create_crawler() - crawler.crawl(spider) + self.crawler_process.crawl(_BenchSpider, total=100000) self.crawler_process.start() diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 40ff9014b..014b00eeb 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -69,20 +69,18 @@ class Command(ScrapyCommand): # contract requests contract_reqs = defaultdict(list) - spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) - spiders = spman_cls.from_settings(self.settings) + spiders = self.crawler_process.spiders - for spider in args or spiders.list(): - spider = spiders.create(spider) - requests = self.get_requests(spider, conman, result) - contract_reqs[spider.name] = [] + for spidername in args or spiders.list(): + spidercls = spiders.load(spidername) + spidercls.start_requests = lambda s: conman.from_spider(s, result) + tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: - for req in requests: - contract_reqs[spider.name].append(req.callback.__name__) - elif requests: - crawler = self.crawler_process.create_crawler(spider.name) - crawler.crawl(spider, requests) + for method in tested_methods: + contract_reqs[spidercls.name].append(method) + elif tested_methods: + self.crawler_process.crawl(spidercls) # start checks if opts.list: @@ -101,15 +99,3 @@ class Command(ScrapyCommand): result.printSummary(start, stop) self.exitcode = int(not result.wasSuccessful()) - def get_requests(self, spider, conman, result): - requests = [] - - for key, value in vars(type(spider)).items(): - if callable(value) and value.__doc__: - bound_method = value.__get__(spider, type(spider)) - request = conman.from_method(bound_method, result) - - if request: - requests.append(request) - - return requests diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 0976de51b..b7fea7b80 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -54,7 +54,5 @@ class Command(ScrapyCommand): raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") spname = args[0] - crawler = self.crawler_process.create_crawler() - spider = crawler.spiders.create(spname, **opts.spargs) - crawler.crawl(spider) + self.crawler_process.crawl(spname, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index b542217e1..e20e7c2e5 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -25,13 +25,12 @@ class Command(ScrapyCommand): if len(args) != 1: raise UsageError() - crawler = self.crawler_process.create_crawler() - editor = crawler.settings['EDITOR'] + editor = self.settings['EDITOR'] try: - spider = crawler.spiders.create(args[0]) + spidercls = self.crawler_process.spiders.load(args[0]) except KeyError: return self._err("Spider not found: %s" % args[0]) - sfile = sys.modules[spider.__module__].__file__ + sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace('.pyc', '.py') self.exitcode = os.system('%s "%s"' % (editor, sfile)) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 373d323c7..ca9fd57f5 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -3,9 +3,8 @@ from w3lib.url import is_url from scrapy.command import ScrapyCommand from scrapy.http import Request -from scrapy.spider import Spider from scrapy.exceptions import UsageError -from scrapy.utils.spider import create_spider_for_request +from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): @@ -48,12 +47,11 @@ class Command(ScrapyCommand): request = Request(args[0], callback=cb, dont_filter=True) request.meta['handle_httpstatus_all'] = True - crawler = self.crawler_process.create_crawler() - spider = None + spidercls = DefaultSpider + spiders = self.crawler_process.spiders if opts.spider: - spider = crawler.spiders.create(opts.spider) + spidercls = spiders.load(opts.spider) else: - spider = create_spider_for_request(crawler.spiders, request, \ - default_spider=Spider('default')) - crawler.crawl(spider, [request]) + spidercls = spidercls_for_request(spiders, request, spidercls) + self.crawler_process.crawl(spidercls, start_requests=lambda: [request]) self.crawler_process.start() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 3e2e24b21..52c5d9f94 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -65,15 +65,14 @@ class Command(ScrapyCommand): return try: - crawler = self.crawler_process.create_crawler() - spider = crawler.spiders.create(name) + spidercls = self.crawler_process.spiders.load(name) except KeyError: pass else: # if spider already exists and not --force then halt if not opts.force: print("Spider %r already exists in module:" % name) - print(" %s" % spider.__module__) + print(" %s" % spidercls.__module__) return template_file = self._find_template(opts.template) if template_file: diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 0ea9c2313..2d55d59bd 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -10,6 +10,5 @@ class Command(ScrapyCommand): return "List available spiders" def run(self, args, opts): - crawler = self.crawler_process.create_crawler() - for s in sorted(crawler.spiders.list()): + for s in sorted(self.crawler_process.spiders.list()): print(s) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 0867a21a0..01c7fff0a 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,7 +5,7 @@ from scrapy.http import Request from scrapy.item import BaseItem from scrapy.utils import display from scrapy.utils.conf import arglist_to_dict -from scrapy.utils.spider import iterate_spider_output, create_spider_for_request +from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError from scrapy import log @@ -113,41 +113,45 @@ class Command(ScrapyCommand): requests.append(x) return items, requests - def get_callback_from_rules(self, response): - if getattr(self.spider, 'rules', None): - for rule in self.spider.rules: + def get_callback_from_rules(self, spider, response): + if getattr(spider, 'rules', None): + for rule in spider.rules: if rule.link_extractor.matches(response.url) and rule.callback: return rule.callback else: log.msg(format='No CrawlSpider rules found in spider %(spider)r, ' 'please specify a callback to use for parsing', - level=log.ERROR, spider=self.spider.name) + level=log.ERROR, spider=spider.name) - def set_spider(self, url, opts): + def set_spidercls(self, url, opts): + spiders = self.crawler_process.spiders if opts.spider: try: - self.spider = self.pcrawler.spiders.create(opts.spider, **opts.spargs) + self.spidercls = spiders.load(opts.spider) except KeyError: log.msg(format='Unable to find spider: %(spider)s', level=log.ERROR, spider=opts.spider) else: - self.spider = create_spider_for_request(self.pcrawler.spiders, Request(url), **opts.spargs) - if not self.spider: + self.spidercls = spidercls_for_request(spiders, Request(url)) + if not self.spidercls: log.msg(format='Unable to find spider for: %(url)s', level=log.ERROR, url=url) - def start_parsing(self, url, opts): request = Request(url, opts.callback) - request = self.prepare_request(request, opts) + _start_requests = lambda s: [self.prepare_request(s, request, opts)] + self.spidercls.start_requests = _start_requests - self.pcrawler.crawl(self.spider, [request]) + + def start_parsing(self, url, opts): + self.crawler_process.crawl(self.spidercls, **opts.spargs) + self.pcrawler = list(self.crawler_process.crawlers)[0] self.crawler_process.start() if not self.first_response: - log.msg(format='No response downloaded for: %(request)s', - level=log.ERROR, request=request) + log.msg(format='No response downloaded for: %(url)s', + level=log.ERROR, url=url) - def prepare_request(self, request, opts): + def prepare_request(self, spider, request, opts): def callback(response): # memorize first request if not self.first_response: @@ -157,17 +161,17 @@ class Command(ScrapyCommand): cb = response.meta['_callback'] if not cb: if opts.rules and self.first_response == response: - cb = self.get_callback_from_rules(response) + cb = self.get_callback_from_rules(spider, response) else: cb = 'parse' if not callable(cb): - cb_method = getattr(self.spider, cb, None) + cb_method = getattr(spider, cb, None) if callable(cb_method): cb = cb_method else: log.msg(format='Cannot find callback %(callback)r in spider: %(spider)s', - callback=callback, spider=self.spider.name, level=log.ERROR) + callback=callback, spider=spider.name, level=log.ERROR) return # parse items and requests @@ -177,7 +181,7 @@ class Command(ScrapyCommand): if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc for item in items: - itemproc.process_item(item, self.spider) + itemproc.process_item(item, spider) self.add_items(depth, items) self.add_requests(depth, requests) @@ -207,10 +211,9 @@ class Command(ScrapyCommand): else: url = args[0] - # prepare spider - self.pcrawler = self.crawler_process.create_crawler() - self.set_spider(url, opts) + # prepare spidercls + self.set_spidercls(url, opts) - if self.spider and opts.depth > 0: + if self.spidercls and opts.depth > 0: self.start_parsing(url, opts) self.print_results(opts) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index b1d501842..b67838619 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -83,8 +83,7 @@ class Command(ScrapyCommand): spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError("No spider found in file: %s\n" % filename) - spider = spclasses.pop()(**opts.spargs) + spidercls = spclasses.pop() - crawler = self.crawler_process.create_crawler() - crawler.crawl(spider) + self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index ab170e665..e2ef1545e 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -8,6 +8,9 @@ from threading import Thread from scrapy.command import ScrapyCommand from scrapy.shell import Shell +from scrapy.http import Request +from scrapy import log +from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): @@ -38,18 +41,31 @@ class Command(ScrapyCommand): pass def run(self, args, opts): - crawler = self.crawler_process.create_crawler() - url = args[0] if args else None - spider = crawler.spiders.create(opts.spider) if opts.spider else None + spiders = self.crawler_process.spiders - self.crawler_process.start_crawling() + spidercls = DefaultSpider + if opts.spider: + spidercls = spiders.load(opts.spider) + elif url: + spidercls = spidercls_for_request(spiders, Request(url), + spidercls, log_multiple=True) + + # The crawler is created this way since the Shell manually handles the + # crawling engine, so the set up in the crawl method won't work + crawler = self.crawler_process._create_logged_crawler(spidercls) + # The Shell class needs a persistent engine in the crawler + crawler.engine = crawler._create_engine() + crawler.engine.start() + + self.crawler_process.start(start_reactor=False) self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url, spider=spider) + shell.start(url=url) def _start_crawler_thread(self): - t = Thread(target=self.crawler_process.start_reactor) + t = Thread(target=self.crawler_process._start_reactor, + kwargs={'stop_after_crawl': False}) t.daemon = True t.start() diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 03e6e4e0c..5eaee3d11 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -15,6 +15,15 @@ class ContractsManager(object): for contract in contracts: self.contracts[contract.name] = contract + def tested_methods_from_spidercls(self, spidercls): + methods = [] + for key, value in vars(spidercls).items(): + if (callable(value) and value.__doc__ and + re.search(r'^\s*@', value.__doc__, re.MULTILINE)): + methods.append(key) + + return methods + def extract_contracts(self, method): contracts = [] for line in method.__doc__.split('\n'): @@ -28,6 +37,14 @@ class ContractsManager(object): return contracts + def from_spider(self, spider, results): + requests = [] + for method in self.tested_methods_from_spidercls(type(spider)): + bound_method = spider.__getattribute__(method) + requests.append(self.from_method(bound_method, results)) + + return requests + def from_method(self, method, results): contracts = self.extract_contracts(method) if contracts: diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index d7e42f6f8..7dc3dacd6 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -86,6 +86,13 @@ class CrawlSpider(Spider): rule.process_links = get_method(rule.process_links) rule.process_request = get_method(rule.process_request) + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs) + spider._follow_links = crawler.settings.getbool( + 'CRAWLSPIDER_FOLLOW_LINKS', True) + return spider + def set_crawler(self, crawler): super(CrawlSpider, self).set_crawler(crawler) self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index cfd6c8003..e0524021a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,4 +1,6 @@ +import six import signal +import warnings from twisted.internet import reactor, defer @@ -6,6 +8,7 @@ from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.extension import ExtensionManager from scrapy.signalmanager import SignalManager +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object from scrapy import log, signals @@ -13,90 +16,100 @@ from scrapy import log, signals class Crawler(object): - def __init__(self, settings): - self.configured = False + def __init__(self, spidercls, settings): + self.spidercls = spidercls self.settings = settings self.signals = SignalManager(self) - self.stats = load_object(settings['STATS_CLASS'])(self) - self._start_requests = lambda: () - self._spider = None - # TODO: move SpiderManager to CrawlerProcess - spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) - self.spiders = spman_cls.from_crawler(self) - - def install(self): - # TODO: remove together with scrapy.project.crawler usage - import scrapy.project - assert not hasattr(scrapy.project, 'crawler'), "crawler already installed" - scrapy.project.crawler = self - - def uninstall(self): - # TODO: remove together with scrapy.project.crawler usage - import scrapy.project - assert hasattr(scrapy.project, 'crawler'), "crawler not installed" - del scrapy.project.crawler - - def configure(self): - if self.configured: - return - - self.configured = True + self.stats = load_object(self.settings['STATS_CLASS'])(self) lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) self.extensions = ExtensionManager.from_crawler(self) - self.engine = ExecutionEngine(self, self._spider_closed) - def crawl(self, spider, requests=None): - assert self._spider is None, 'Spider already attached' - self._spider = spider - spider.set_crawler(self) - if requests is None: - self._start_requests = spider.start_requests - else: - self._start_requests = lambda: requests + self.crawling = False + self.spider = None + self.engine = None - def _spider_closed(self, spider=None): - if not self.engine.open_spiders: - self.stop() + @property + def spiders(self): + if not hasattr(self, '_spiders'): + warnings.warn("Crawler.spiders is deprecated, use " + "CrawlerRunner.spiders or instantiate " + "scrapy.spidermanager.SpiderManager with your " + "settings.", + category=ScrapyDeprecationWarning, stacklevel=2) + spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) + self._spiders = spman_cls.from_settings(self.settings) + return self._spiders @defer.inlineCallbacks - def start(self): - yield defer.maybeDeferred(self.configure) - if self._spider: - yield self.engine.open_spider(self._spider, self._start_requests()) - yield defer.maybeDeferred(self.engine.start) + def crawl(self, *args, **kwargs): + assert not self.crawling, "Crawling already taking place" + self.crawling = True + + try: + self.spider = self._create_spider(*args, **kwargs) + self.engine = self._create_engine() + start_requests = iter(self.spider.start_requests()) + yield self.engine.open_spider(self.spider, start_requests) + yield defer.maybeDeferred(self.engine.start) + except Exception: + self.crawling = False + raise + + def _create_spider(self, *args, **kwargs): + return self.spidercls.from_crawler(self, *args, **kwargs) + + def _create_engine(self): + return ExecutionEngine(self, lambda _: self.stop()) @defer.inlineCallbacks def stop(self): - if self.configured and self.engine.running: + if self.crawling: + self.crawling = False yield defer.maybeDeferred(self.engine.stop) -class CrawlerProcess(object): - """ A class to run multiple scrapy crawlers in a process sequentially""" +class CrawlerRunner(object): def __init__(self, settings): - install_shutdown_handlers(self._signal_shutdown) self.settings = settings - self.crawlers = {} - self.stopping = False - self._started = None + smcls = load_object(settings['SPIDER_MANAGER_CLASS']) + self.spiders = smcls.from_settings(settings.frozencopy()) + self.crawlers = set() + self.crawl_deferreds = set() - def create_crawler(self, name=None): - if name not in self.crawlers: - self.crawlers[name] = Crawler(self.settings) + def crawl(self, spidercls, *args, **kwargs): + crawler = self._create_logged_crawler(spidercls) + self.crawlers.add(crawler) - return self.crawlers[name] + d = crawler.crawl(*args, **kwargs) + self.crawl_deferreds.add(d) + return d - def start(self): - if self.start_crawling(): - self.start_reactor() + def _create_logged_crawler(self, spidercls): + crawler = self._create_crawler(spidercls) + log_observer = log.start_from_crawler(crawler) + if log_observer: + crawler.signals.connect(log_observer.stop, signals.engine_stopped) + return crawler + + def _create_crawler(self, spidercls): + if isinstance(spidercls, six.string_types): + spidercls = self.spiders.load(spidercls) + crawler = Crawler(spidercls, self.settings.frozencopy()) + return crawler - @defer.inlineCallbacks def stop(self): - self.stopping = True - if self._active_crawler: - yield self._active_crawler.stop() + return defer.DeferredList(c.stop() for c in self.crawlers) + + +class CrawlerProcess(CrawlerRunner): + """A class to run multiple scrapy crawlers in a process simultaneously""" + + def __init__(self, settings): + super(CrawlerProcess, self).__init__(settings) + install_shutdown_handlers(self._signal_shutdown) + self.stopping = False def _signal_shutdown(self, signum, _): install_shutdown_handlers(self._signal_kill) @@ -110,44 +123,29 @@ class CrawlerProcess(object): signame = signal_names[signum] log.msg(format='Received %(signame)s twice, forcing unclean shutdown', level=log.INFO, signame=signame) + self._stop_logging() reactor.callFromThread(self._stop_reactor) - # ------------------------------------------------------------------------# - # The following public methods can't be considered stable and may change at - # any moment. - # - # start_crawling and start_reactor are called from scrapy.commands.shell - # They are splitted because reactor is started on a different thread than IPython shell. - # - def start_crawling(self): + def start(self, stop_after_crawl=True, start_reactor=True): + self.log_observer = log.start_from_settings(self.settings) log.scrapy_info(self.settings) - return self._start_crawler() is not None + if start_reactor: + self._start_reactor(stop_after_crawl) - def start_reactor(self): + def _start_reactor(self, stop_after_crawl=True): + if stop_after_crawl: + d = defer.DeferredList(self.crawl_deferreds) + if d.called: + # Don't start the reactor if the deferreds are already fired + return + d.addBoth(lambda _: self._stop_reactor()) if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call - def _start_crawler(self): - if not self.crawlers or self.stopping: - return - - name, crawler = self.crawlers.popitem() - self._active_crawler = crawler - log_observer = log.start_from_crawler(crawler) - crawler.configure() - crawler.install() - crawler.signals.connect(crawler.uninstall, signals.engine_stopped) - if log_observer: - crawler.signals.connect(log_observer.stop, signals.engine_stopped) - crawler.signals.connect(self._check_done, signals.engine_stopped) - crawler.start() - return name, crawler - - def _check_done(self, **kwargs): - if not self._start_crawler(): - self._stop_reactor() + def _stop_logging(self): + self.log_observer.stop() def _stop_reactor(self, _=None): try: diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index 5d8d85aca..d4596407e 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -2,10 +2,12 @@ from zope.interface import Interface class ISpiderManager(Interface): - def create(spider_name, **spider_args): - """Returns a new Spider instance for the given spider name, and using - the given spider arguments. If the spider name is not found, it must - raise a KeyError.""" + def from_settings(settings): + """Returns an instance of the class for the given settings""" + + def load(spider_name): + """Returns the Spider class for the given spider name. If the spider + name is not found, it must raise a KeyError.""" def list(): """Return a list with the names of all spiders available in the diff --git a/scrapy/log.py b/scrapy/log.py index 1f32003e7..aa53e3574 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -35,15 +35,16 @@ class ScrapyFileLogObserver(log.FileLogObserver): def __init__(self, f, level=INFO, encoding='utf-8', crawler=None): self.level = level self.encoding = encoding + self.crawler = crawler if crawler: - self.crawler = crawler self.emit = self._emit_with_crawler else: self.emit = self._emit log.FileLogObserver.__init__(self, f) def _emit(self, eventDict): - ev = _adapt_eventdict(eventDict, self.level, self.encoding) + ev = _adapt_eventdict(eventDict, self.level, self.encoding, + self.crawler) if ev is not None: log.FileLogObserver.emit(self, ev) return ev @@ -55,7 +56,8 @@ class ScrapyFileLogObserver(log.FileLogObserver): sname = 'log_count/%s' % level_names.get(level, level) self.crawler.stats.inc_value(sname) -def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', prepend_level=True): +def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', + crawler=None, prepend_level=True): """Adapt Twisted log eventDict making it suitable for logging with a Scrapy log observer. It may return None to indicate that the event should be ignored by a Scrapy log observer. @@ -78,6 +80,12 @@ def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', prepend_level= spider = ev.get('spider') if spider: ev['system'] = unicode_to_str(spider.name, encoding) + if crawler and (not spider or spider.crawler is not crawler): + # ignore events not triggered by own spiders in crawlers' observers + return + if not crawler and spider: + # ignore spiders' events in observers without crawler + return lvlname = level_names.get(level, 'NOLEVEL') message = ev.get('message') @@ -140,18 +148,14 @@ def start_from_settings(settings, crawler=None): settings['LOG_ENCODING'], crawler) def scrapy_info(settings): - log_observer = start_from_settings(settings) - if log_observer: - msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, - settings['BOT_NAME'])) + msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, + settings['BOT_NAME'])) - msg("Optional features available: %s" % ", ".join(scrapy.optional_features), - level=INFO) + msg("Optional features available: %s" % ", ".join(scrapy.optional_features), + level=INFO) - d = dict(overridden_settings(settings)) - msg(format="Overridden settings: %(settings)r", settings=d, level=INFO) - - log_observer.stop() + d = dict(overridden_settings(settings)) + msg(format="Overridden settings: %(settings)r", settings=d, level=INFO) def start_from_crawler(crawler): return start_from_settings(crawler.settings, crawler) diff --git a/scrapy/project.py b/scrapy/project.py index bbe947761..d8973a6c7 100644 --- a/scrapy/project.py +++ b/scrapy/project.py @@ -1,13 +1,17 @@ -""" ---------- WARNING: THIS MODULE IS DEPRECATED ----------- -This module is deprecated. If you want to get the Scrapy crawler from your -extension, middleware or pipeline implement the `from_crawler` class method. +""" +Obsolete module, kept for giving a meaningful error message when trying to +import. +""" + +raise ImportError("""scrapy.project usage has become obsolete. + +If you want to get the Scrapy crawler from your extension, middleware or +pipeline implement the `from_crawler` class method (or look up for extending +components that have already done it, such as spiders). For example: @classmethod def from_crawler(cls, crawler): - return cls(crawler) - -""" + return cls(crawler)""") diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 2dd6a2920..938b93564 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,5 +1,6 @@ import six import json +import copy import warnings from collections import MutableMapping from importlib import import_module @@ -46,6 +47,7 @@ class SettingsAttribute(object): class Settings(object): def __init__(self, values=None, priority='project'): + self.frozen = False self.attributes = {} self.setmodule(default_settings, priority='default') if values is not None: @@ -74,25 +76,19 @@ class Settings(object): return float(self.get(name, default)) def getlist(self, name, default=None): - value = self.get(name) - if value is None: - return default or [] - elif hasattr(value, '__iter__'): - return value - else: - return str(value).split(',') + value = self.get(name, default or []) + if isinstance(value, six.string_types): + value = value.split(',') + return list(value) def getdict(self, name, default=None): - value = self.get(name) - if value is None: - return default or {} + value = self.get(name, default or {}) if isinstance(value, six.string_types): value = json.loads(value) - if isinstance(value, dict): - return value - raise ValueError("Cannot convert value for setting '%s' to dict: '%s'" % (name, value)) + return dict(value) def set(self, name, value, priority='project'): + self._assert_mutability() if isinstance(priority, six.string_types): priority = SETTINGS_PRIORITIES[priority] if name not in self.attributes: @@ -101,16 +97,33 @@ class Settings(object): self.attributes[name].set(value, priority) def setdict(self, values, priority='project'): + self._assert_mutability() for name, value in six.iteritems(values): self.set(name, value, priority) def setmodule(self, module, priority='project'): + self._assert_mutability() if isinstance(module, six.string_types): module = import_module(module) for key in dir(module): if key.isupper(): self.set(key, getattr(module, key), priority) + def _assert_mutability(self): + if self.frozen: + raise TypeError("Trying to modify an immutable Settings object") + + def copy(self): + return copy.deepcopy(self) + + def freeze(self): + self.frozen = True + + def frozencopy(self): + copy = self.copy() + copy.freeze() + return copy + @property def overrides(self): warnings.warn("`Settings.overrides` attribute is deprecated and won't " diff --git a/scrapy/shell.py b/scrapy/shell.py index 74eaef40f..8f87fcb41 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -21,7 +21,6 @@ from scrapy.spider import Spider from scrapy.utils.console import start_python_console from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser -from scrapy.utils.spider import create_spider_for_request class Shell(object): @@ -67,11 +66,9 @@ class Shell(object): return self.spider if spider is None: - spider = create_spider_for_request(self.crawler.spiders, - request, - Spider('default'), - log_multiple=True) - spider.set_crawler(self.crawler) + spider = self.crawler.spider or self.crawler._create_spider() + + self.crawler.spider = spider self.crawler.engine.open_spider(spider, close_if_idle=False) self.spider = spider return spider @@ -126,10 +123,9 @@ class Shell(object): return isinstance(value, self.relevant_classes) -def inspect_response(response, spider=None): +def inspect_response(response, spider): """Open a shell to inspect the given response""" - from scrapy.project import crawler - Shell(crawler).start(response=response, spider=spider) + Shell(spider.crawler).start(response=response) def _request_deferred(request): diff --git a/scrapy/spider.py b/scrapy/spider.py index 8ecfae2a0..943925042 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -3,11 +3,15 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ +import warnings + from scrapy import log +from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider from scrapy.utils.deprecate import create_deprecated_class +from scrapy.exceptions import ScrapyDeprecationWarning class Spider(object_ref): @@ -32,18 +36,25 @@ class Spider(object_ref): """ log.msg(message, spider=self, level=level, **kw) + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(*args, **kwargs) + spider._set_crawler(crawler) + return spider + def set_crawler(self, crawler): - assert not hasattr(self, '_crawler'), "Spider already bounded to %s" % crawler - self._crawler = crawler + warnings.warn("set_crawler is deprecated, instantiate and bound the " + "spider to this crawler with from_crawler method " + "instead.", + category=ScrapyDeprecationWarning, stacklevel=2) + assert not hasattr(self, 'crawler'), "Spider already bounded to a " \ + "crawler" + self._set_crawler(crawler) - @property - def crawler(self): - assert hasattr(self, '_crawler'), "Spider not bounded to any crawler" - return self._crawler - - @property - def settings(self): - return self.crawler.settings + def _set_crawler(self, crawler): + self.crawler = crawler + self.settings = crawler.settings + crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self): for url in self.start_urls: @@ -59,6 +70,12 @@ class Spider(object_ref): def handles_request(cls, request): return url_is_from_spider(request.url, cls) + @staticmethod + def close(spider, reason): + closed = getattr(spider, 'closed', None) + if callable(closed): + return closed(reason) + def __str__(self): return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self)) @@ -76,6 +93,6 @@ class ObsoleteClass(object): raise AttributeError(self.message) spiders = ObsoleteClass(""" -"from scrapy.spider import spiders" no longer works - use "from scrapy.project import crawler" and then access crawler.spiders attribute" +"from scrapy.spider import spiders" no longer works - use "from scrapy.spidermanager import SpiderManager" and instantiate it with your project settings" """) diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py index 5a0951cb4..5715b7793 100644 --- a/scrapy/spidermanager.py +++ b/scrapy/spidermanager.py @@ -6,7 +6,6 @@ spiders from zope.interface import implementer import six -from scrapy import signals from scrapy.interfaces import ISpiderManager from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes @@ -15,8 +14,8 @@ from scrapy.utils.spider import iter_spider_classes @implementer(ISpiderManager) class SpiderManager(object): - def __init__(self, spider_modules): - self.spider_modules = spider_modules + def __init__(self, settings): + self.spider_modules = settings['SPIDER_MODULES'] self._spiders = {} for name in self.spider_modules: for module in walk_modules(name): @@ -28,33 +27,17 @@ class SpiderManager(object): @classmethod def from_settings(cls, settings): - return cls(settings.getlist('SPIDER_MODULES')) + return cls(settings) - @classmethod - def from_crawler(cls, crawler): - sm = cls.from_settings(crawler.settings) - sm.crawler = crawler - crawler.signals.connect(sm.close_spider, signals.spider_closed) - return sm - - def create(self, spider_name, **spider_kwargs): + def load(self, spider_name): try: - spcls = self._spiders[spider_name] + return self._spiders[spider_name] except KeyError: - raise KeyError("Spider not found: %s" % spider_name) - if hasattr(self, 'crawler') and hasattr(spcls, 'from_crawler'): - return spcls.from_crawler(self.crawler, **spider_kwargs) - else: - return spcls(**spider_kwargs) + raise KeyError("Spider not found: {}".format(spider_name)) def find_by_request(self, request): return [name for name, cls in six.iteritems(self._spiders) if cls.handles_request(request)] def list(self): - return self._spiders.keys() - - def close_spider(self, spider, reason): - closed = getattr(spider, 'closed', None) - if callable(closed): - return closed(reason) + return list(self._spiders.keys()) diff --git a/scrapy/stats.py b/scrapy/stats.py index b8128dfc2..710601430 100644 --- a/scrapy/stats.py +++ b/scrapy/stats.py @@ -1,7 +1,8 @@ -from scrapy.project import crawler -stats = crawler.stats -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.stats` is deprecated, use `crawler.stats` attribute instead", - ScrapyDeprecationWarning, stacklevel=2) +""" +Obsolete module, kept for giving a meaningful error message when trying to +import. +""" + +raise ImportError("scrapy.stats usage has become obsolete, use " + "`crawler.stats` attribute instead") diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 4e43bc13f..b81cf2b9b 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,6 +4,7 @@ import six from scrapy import log from scrapy.item import BaseItem +from scrapy.spider import Spider from scrapy.utils.misc import arg_to_iter @@ -25,21 +26,21 @@ def iter_spider_classes(module): getattr(obj, 'name', None): yield obj -def create_spider_for_request(spidermanager, request, default_spider=None, \ - log_none=False, log_multiple=False, **spider_kwargs): - """Create a spider to handle the given Request. +def spidercls_for_request(spidermanager, request, default_spidercls=None, + log_none=False, log_multiple=False): + """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using - the spider manager) and return a (new) Spider if (and only if) there is + the spider manager) and return a Spider class if (and only if) there is only one Spider able to handle the Request. If multiple spiders (or no spider) are found, it will return the - default_spider passed. It can optionally log if multiple or no spiders + default_spidercls passed. It can optionally log if multiple or no spiders are found. """ snames = spidermanager.find_by_request(request) if len(snames) == 1: - return spidermanager.create(snames[0], **spider_kwargs) + return spidermanager.load(snames[0]) if len(snames) > 1 and log_multiple: log.msg(format='More than one spider can handle: %(request)s - %(snames)s', @@ -49,5 +50,8 @@ def create_spider_for_request(spidermanager, request, default_spider=None, \ log.msg(format='Unable to find spider that handles: %(request)s', level=log.ERROR, request=request) - return default_spider + return default_spidercls + +class DefaultSpider(Spider): + name = 'default' diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index e6376d519..a4b769970 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -20,15 +20,17 @@ def assert_aws_environ(): if 'AWS_ACCESS_KEY_ID' not in os.environ: raise SkipTest("AWS keys not found") -def get_crawler(settings_dict=None): +def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. """ - from scrapy.crawler import Crawler + from scrapy.crawler import CrawlerRunner from scrapy.settings import Settings + from scrapy.spider import Spider - return Crawler(Settings(settings_dict)) + runner = CrawlerRunner(Settings(settings_dict)) + return runner._create_crawler(spidercls or Spider) def get_pythonpath(): """Return a PYTHONPATH suitable to use in processes so that they find this @@ -62,10 +64,3 @@ def assert_samelines(testcase, text1, text2, msg=None): line endings between platforms """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) - -def docrawl(spider, settings=None): - """Configure and start Crawler; return the result of crawler.start()""" - crawler = get_crawler(settings) - crawler.configure() - crawler.crawl(spider) - return crawler.start() diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index c1619b3ae..f3c667cd0 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -8,12 +8,14 @@ tests/test_contrib_exporter.py tests/test_contrib_linkextractors.py tests/test_contrib_loader.py tests/test_crawl.py +tests/test_crawler.py tests/test_djangoitem/__init__.py tests/test_downloader_handlers.py tests/test_downloadermiddleware_ajaxcrawlable.py tests/test_downloadermiddleware_cookies.py tests/test_downloadermiddleware_decompression.py tests/test_downloadermiddleware_defaultheaders.py +tests/test_downloadermiddleware_downloadtimeout.py tests/test_downloadermiddleware_httpauth.py tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py @@ -22,6 +24,7 @@ tests/test_downloadermiddleware.py tests/test_downloadermiddleware_redirect.py tests/test_downloadermiddleware_retry.py tests/test_downloadermiddleware_robotstxt.py +tests/test_downloadermiddleware_stats.py tests/test_downloadermiddleware_useragent.py tests/test_dupefilter.py tests/test_engine.py @@ -48,9 +51,12 @@ tests/test_spidermanager/test_spiders/spider1.py tests/test_spidermanager/test_spiders/spider2.py tests/test_spidermanager/test_spiders/spider3.py tests/test_spidermanager/test_spiders/spider4.py +tests/test_spidermiddleware_depth.py tests/test_spidermiddleware_httperror.py +tests/test_spidermiddleware_offsite.py tests/test_spidermiddleware_referer.py tests/test_spider.py +tests/test_stats.py tests/test_utils_defer.py tests/test_utils_iterators.py tests/test_utils_jsonrpc.py diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 8d30a4643..1700a861e 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,6 +1,6 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.utils.test import docrawl +from scrapy.utils.test import get_crawler from tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider from tests.mockserver import MockServer @@ -16,45 +16,45 @@ class TestCloseSpider(TestCase): @defer.inlineCallbacks def test_closespider_itemcount(self): - spider = ItemSpider() close_on = 5 - yield docrawl(spider, {'CLOSESPIDER_ITEMCOUNT': close_on}) - reason = spider.meta['close_reason'] + crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on}) + yield crawler.crawl() + reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_itemcount') - itemcount = spider.crawler.stats.get_value('item_scraped_count') + itemcount = crawler.stats.get_value('item_scraped_count') self.assertTrue(itemcount >= close_on) @defer.inlineCallbacks def test_closespider_pagecount(self): - spider = FollowAllSpider() close_on = 5 - yield docrawl(spider, {'CLOSESPIDER_PAGECOUNT': close_on}) - reason = spider.meta['close_reason'] + crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on}) + yield crawler.crawl() + reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_pagecount') - pagecount = spider.crawler.stats.get_value('response_received_count') + pagecount = crawler.stats.get_value('response_received_count') self.assertTrue(pagecount >= close_on) @defer.inlineCallbacks def test_closespider_errorcount(self): - spider = ErrorSpider(total=1000000) close_on = 5 - yield docrawl(spider, {'CLOSESPIDER_ERRORCOUNT': close_on}) - self.flushLoggedErrors(spider.exception_cls) - reason = spider.meta['close_reason'] + crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on}) + yield crawler.crawl(total=1000000) + self.flushLoggedErrors(crawler.spider.exception_cls) + reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') key = 'spider_exceptions/{name}'\ - .format(name=spider.exception_cls.__name__) - errorcount = spider.crawler.stats.get_value(key) + .format(name=crawler.spider.exception_cls.__name__) + errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) @defer.inlineCallbacks def test_closespider_timeout(self): - spider = FollowAllSpider(total=1000000) close_on = 0.1 - yield docrawl(spider, {'CLOSESPIDER_TIMEOUT': close_on}) - reason = spider.meta['close_reason'] + crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on}) + yield crawler.crawl(total=1000000) + reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_timeout') - stats = spider.crawler.stats + stats = crawler.stats start = stats.get_value('start_time') stop = stats.get_value('finish_time') diff = stop - start diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 9401bd0c9..48931d6ff 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -3,7 +3,7 @@ import socket import mock from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.utils.test import docrawl, get_testlog +from scrapy.utils.test import get_crawler, get_testlog from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider from tests.mockserver import MockServer @@ -21,9 +21,9 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_follow_all(self): - spider = FollowAllSpider() - yield docrawl(spider) - self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url + crawler = get_crawler(FollowAllSpider) + yield crawler.crawl() + self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks def test_delay(self): @@ -37,9 +37,9 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def _test_delay(self, delay, randomize): settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} - spider = FollowAllSpider(maxlatency=delay * 2) - yield docrawl(spider, settings) - t = spider.times + crawler = get_crawler(FollowAllSpider, settings) + yield crawler.crawl(maxlatency=delay * 2) + t = crawler.spider.times totaltime = t[-1] - t[0] avgd = totaltime / (len(t) - 1) tolerance = 0.6 if randomize else 0.2 @@ -48,85 +48,79 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_success(self): - spider = DelaySpider(n=0.5) - yield docrawl(spider) - self.assertTrue(spider.t1 > 0) - self.assertTrue(spider.t2 > 0) - self.assertTrue(spider.t2 > spider.t1) + crawler = get_crawler(DelaySpider) + yield crawler.crawl(n=0.5) + self.assertTrue(crawler.spider.t1 > 0) + self.assertTrue(crawler.spider.t2 > 0) + self.assertTrue(crawler.spider.t2 > crawler.spider.t1) @defer.inlineCallbacks def test_timeout_failure(self): - spider = DelaySpider(n=0.5) - yield docrawl(spider, {"DOWNLOAD_TIMEOUT": 0.35}) - self.assertTrue(spider.t1 > 0) - self.assertTrue(spider.t2 == 0) - self.assertTrue(spider.t2_err > 0) - self.assertTrue(spider.t2_err > spider.t1) + crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) + yield crawler.crawl(n=0.5) + self.assertTrue(crawler.spider.t1 > 0) + self.assertTrue(crawler.spider.t2 == 0) + self.assertTrue(crawler.spider.t2_err > 0) + self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) # server hangs after receiving response headers - spider = DelaySpider(n=0.5, b=1) - yield docrawl(spider, {"DOWNLOAD_TIMEOUT": 0.35}) - self.assertTrue(spider.t1 > 0) - self.assertTrue(spider.t2 == 0) - self.assertTrue(spider.t2_err > 0) - self.assertTrue(spider.t2_err > spider.t1) + yield crawler.crawl(n=0.5, b=1) + self.assertTrue(crawler.spider.t1 > 0) + self.assertTrue(crawler.spider.t2 == 0) + self.assertTrue(crawler.spider.t2_err > 0) + self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) @defer.inlineCallbacks def test_retry_503(self): - spider = SimpleSpider("http://localhost:8998/status?n=503") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://localhost:8998/status?n=503") self._assert_retried() @defer.inlineCallbacks def test_retry_conn_failed(self): - spider = SimpleSpider("http://localhost:65432/status?n=503") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://localhost:65432/status?n=503") self._assert_retried() @defer.inlineCallbacks def test_retry_dns_error(self): with mock.patch('socket.gethostbyname', side_effect=socket.gaierror(-5, 'No address associated with hostname')): - spider = SimpleSpider("http://example.com/") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://example.com/") self._assert_retried() @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): - spider = BrokenStartRequestsSpider(fail_before_yield=1) - yield docrawl(spider) + crawler = get_crawler(BrokenStartRequestsSpider) + yield crawler.crawl(fail_before_yield=1) errors = self.flushLoggedErrors(ZeroDivisionError) self.assertEqual(len(errors), 1) @defer.inlineCallbacks def test_start_requests_bug_yielding(self): - spider = BrokenStartRequestsSpider(fail_yielding=1) - yield docrawl(spider) + crawler = get_crawler(BrokenStartRequestsSpider) + yield crawler.crawl(fail_yielding=1) errors = self.flushLoggedErrors(ZeroDivisionError) self.assertEqual(len(errors), 1) @defer.inlineCallbacks def test_start_requests_lazyness(self): settings = {"CONCURRENT_REQUESTS": 1} - spider = BrokenStartRequestsSpider() - yield docrawl(spider, settings) - #self.assertTrue(False, spider.seedsseen) - #self.assertTrue(spider.seedsseen.index(None) < spider.seedsseen.index(99), - # spider.seedsseen) + crawler = get_crawler(BrokenStartRequestsSpider, settings) + yield crawler.crawl() + #self.assertTrue(False, crawler.spider.seedsseen) + #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), + # crawler.spider.seedsseen) @defer.inlineCallbacks def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - spider = DuplicateStartRequestsSpider(dont_filter=True, - distinct_urls=2, - dupe_factor=3) - yield docrawl(spider, settings) - self.assertEqual(spider.visited, 6) + crawler = get_crawler(DuplicateStartRequestsSpider, settings) + yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3) + self.assertEqual(crawler.spider.visited, 6) - spider = DuplicateStartRequestsSpider(dont_filter=False, - distinct_urls=3, - dupe_factor=4) - yield docrawl(spider, settings) - self.assertEqual(spider.visited, 3) + yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4) + self.assertEqual(crawler.spider.visited, 3) @defer.inlineCallbacks def test_unbounded_response(self): @@ -150,23 +144,23 @@ Connection: close foo body with multiples lines '''}) - spider = SimpleSpider("http://localhost:8998/raw?{0}".format(query)) - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://localhost:8998/raw?{0}".format(query)) log = get_testlog() self.assertEqual(log.count("Got response 200"), 1) @defer.inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data - spider = SimpleSpider("http://localhost:8998/drop?abort=0") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://localhost:8998/drop?abort=0") self._assert_retried() @defer.inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data - spider = SimpleSpider("http://localhost:8998/drop?abort=1") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://localhost:8998/drop?abort=1") self._assert_retried() def _assert_retried(self): @@ -184,22 +178,22 @@ with multiples lines req0.meta['next'] = req1 req1.meta['next'] = req2 req2.meta['next'] = req3 - spider = SingleRequestSpider(seed=req0) - yield docrawl(spider) + crawler = get_crawler(SingleRequestSpider) + yield crawler.crawl(seed=req0) # basic asserts in case of weird communication errors - self.assertIn('responses', spider.meta) - self.assertNotIn('failures', spider.meta) + self.assertIn('responses', crawler.spider.meta) + self.assertNotIn('failures', crawler.spider.meta) # start requests doesn't set Referer header - echo0 = json.loads(spider.meta['responses'][2].body) + echo0 = json.loads(crawler.spider.meta['responses'][2].body) self.assertNotIn('Referer', echo0['headers']) # following request sets Referer to start request url - echo1 = json.loads(spider.meta['responses'][1].body) + echo1 = json.loads(crawler.spider.meta['responses'][1].body) self.assertEqual(echo1['headers'].get('Referer'), [req0.url]) # next request avoids Referer header - echo2 = json.loads(spider.meta['responses'][2].body) + echo2 = json.loads(crawler.spider.meta['responses'][2].body) self.assertNotIn('Referer', echo2['headers']) # last request explicitly sets a Referer header - echo3 = json.loads(spider.meta['responses'][3].body) + echo3 = json.loads(crawler.spider.meta['responses'][3].body) self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com']) @defer.inlineCallbacks @@ -208,11 +202,11 @@ with multiples lines est = [] def cb(response): - est.append(get_engine_status(spider.crawler.engine)) + est.append(get_engine_status(crawler.engine)) - spider = SingleRequestSpider(seed='http://localhost:8998/', callback_func=cb) - yield docrawl(spider) + crawler = get_crawler(SingleRequestSpider) + yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb) self.assertEqual(len(est), 1, est) s = dict(est[0]) - self.assertEqual(s['engine.spider.name'], spider.name) + self.assertEqual(s['engine.spider.name'], crawler.spider.name) self.assertEqual(s['len(engine.scraper.slot.active)'], 1) diff --git a/tests/test_crawler.py b/tests/test_crawler.py new file mode 100644 index 000000000..55381c030 --- /dev/null +++ b/tests/test_crawler.py @@ -0,0 +1,24 @@ +import warnings +import unittest + +from scrapy.crawler import Crawler +from scrapy.settings import Settings +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.misc import load_object + + +class CrawlerTestCase(unittest.TestCase): + + def setUp(self): + self.crawler = Crawler(DefaultSpider, Settings()) + + def test_deprecated_attribute_spiders(self): + with warnings.catch_warnings(record=True) as w: + spiders = self.crawler.spiders + self.assertEqual(len(w), 1) + self.assertIn("Crawler.spiders", str(w[0].message)) + sm_cls = load_object(self.crawler.settings['SPIDER_MANAGER_CLASS']) + self.assertIsInstance(spiders, sm_cls) + + self.crawler.spiders + self.assertEqual(len(w), 1, "Warn deprecated access only once") diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 6a3115004..c444d35fa 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -47,19 +47,22 @@ class LoadTestCase(unittest.TestCase): def test_enabled_handler(self): handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'} - dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._handlers) self.assertNotIn('scheme', dh._notconfigured) def test_not_configured_handler(self): handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'} - dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + dh = DownloadHandlers(crawler) self.assertNotIn('scheme', dh._handlers) self.assertIn('scheme', dh._notconfigured) def test_disabled_handler(self): handlers = {'scheme': None} - dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + dh = DownloadHandlers(crawler) self.assertNotIn('scheme', dh._handlers) self.assertNotIn('scheme', dh._notconfigured) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index b7d3594cd..282035f5c 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -12,9 +12,8 @@ class ManagerTestCase(TestCase): settings_dict = None def setUp(self): - self.crawler = get_crawler(self.settings_dict) - self.spider = Spider('foo') - self.spider.set_crawler(self.crawler) + self.crawler = get_crawler(Spider, self.settings_dict) + self.spider = self.crawler._create_spider('foo') self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) # some mw depends on stats collector self.crawler.stats.open_spider(self.spider) diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 3e5ce6052..e73e62538 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -9,8 +9,8 @@ __doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawl'] class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): - self.spider = Spider('foo') - crawler = get_crawler({'AJAXCRAWL_ENABLED': True}) + crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True}) + self.spider = crawler._create_spider('foo') self.mw = AjaxCrawlMiddleware.from_crawler(crawler) def _ajaxcrawlable_body(self): diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index b37a02336..09973b367 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -10,9 +10,8 @@ from scrapy.utils.test import get_crawler class TestDefaultHeadersMiddleware(TestCase): def get_defaults_spider_mw(self): - crawler = get_crawler() - spider = Spider('foo') - spider.set_crawler(crawler) + crawler = get_crawler(Spider) + spider = crawler._create_spider('foo') defaults = dict([(k, [v]) for k, v in \ six.iteritems(crawler.settings.get('DEFAULT_REQUEST_HEADERS'))]) return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler) diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 52a0cc09d..3e3ff2401 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -9,9 +9,8 @@ from scrapy.utils.test import get_crawler class DownloadTimeoutMiddlewareTest(unittest.TestCase): def get_request_spider_mw(self): - crawler = get_crawler() - spider = Spider('foo') - spider.set_crawler(crawler) + crawler = get_crawler(Spider) + spider = crawler._create_spider('foo') request = Request('http://scrapytest.org/') return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 0eb5e7144..1e22ae661 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -24,8 +24,8 @@ class _BaseTest(unittest.TestCase): self.yesterday = email.utils.formatdate(time.time() - 86400) self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) - self.crawler = get_crawler() - self.spider = Spider('example.com') + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('example.com') self.tmpdir = tempfile.mkdtemp() self.request = Request('http://www.example.com', headers={'User-Agent': 'test'}) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 9673d4594..fb70b13ae 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -10,8 +10,8 @@ from scrapy.utils.test import get_crawler class RedirectMiddlewareTest(unittest.TestCase): def setUp(self): - crawler = get_crawler() - self.spider = Spider('foo') + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') self.mw = RedirectMiddleware.from_crawler(crawler) def test_priority_adjust(self): @@ -132,8 +132,8 @@ class RedirectMiddlewareTest(unittest.TestCase): class MetaRefreshMiddlewareTest(unittest.TestCase): def setUp(self): - crawler = get_crawler() - self.spider = Spider('foo') + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') self.mw = MetaRefreshMiddleware.from_crawler(crawler) def _body(self, interval=5, url='http://example.org/newpage'): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 166c2bff6..669abea05 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -14,8 +14,8 @@ from scrapy.utils.test import get_crawler class RetryTest(unittest.TestCase): def setUp(self): - crawler = get_crawler() - self.spider = Spider('foo') + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') self.mw = RetryMiddleware.from_crawler(crawler) self.mw.max_retry_times = 2 diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index edc26e543..b790ff09a 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -9,8 +9,8 @@ from scrapy.utils.test import get_crawler class TestDownloaderStats(TestCase): def setUp(self): - self.crawler = get_crawler() - self.spider = Spider('scrapytest.org') + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('scrapytest.org') self.mw = DownloaderStats(self.crawler.stats) self.crawler.stats.open_spider(self.spider) diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 5fd5c24be..909d03ba5 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -9,9 +9,8 @@ from scrapy.utils.test import get_crawler class UserAgentMiddlewareTest(TestCase): def get_spider_and_mw(self, default_useragent): - crawler = get_crawler({'USER_AGENT': default_useragent}) - spider = Spider('foo') - spider.set_crawler(crawler) + crawler = get_crawler(Spider, {'USER_AGENT': default_useragent}) + spider = crawler._create_spider('foo') return spider, UserAgentMiddleware.from_crawler(crawler) def test_default_agent(self): diff --git a/tests/test_engine.py b/tests/test_engine.py index 6a0314a02..67fb8ae79 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -87,20 +87,17 @@ class CrawlerRun(object): self.portno = self.port.getHost().port start_urls = [self.geturl("/"), self.geturl("/redirect")] - self.spider = TestSpider(start_urls=start_urls) for name, signal in vars(signals).items(): if not name.startswith('_'): dispatcher.connect(self.record_signal, signal) - self.crawler = get_crawler() - self.crawler.install() - self.crawler.configure() + self.crawler = get_crawler(TestSpider) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) - self.crawler.crawl(self.spider) - self.crawler.start() + self.crawler.crawl(start_urls=start_urls) + self.spider = self.crawler.spider self.deferred = defer.Deferred() dispatcher.connect(self.stop, signals.engine_stopped) @@ -111,7 +108,6 @@ class CrawlerRun(object): for name, signal in vars(signals).items(): if not name.startswith('_'): disconnect_all(signal) - self.crawler.uninstall() self.deferred.callback(None) def geturl(self, path): diff --git a/tests/test_log.py b/tests/test_log.py index 3263cb42e..113d10004 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -6,6 +6,7 @@ from twisted.trial import unittest from scrapy import log from scrapy.spider import Spider from scrapy.settings import default_settings +from scrapy.utils.test import get_crawler class LogTest(unittest.TestCase): @@ -40,10 +41,10 @@ class ScrapyFileLogObserverTest(unittest.TestCase): log.msg("Hello") self.assertEqual(self.logged(), "[scrapy] INFO: Hello") - def test_msg_spider(self): + def test_msg_ignore_spider(self): spider = Spider("myspider") log.msg("Hello", spider=spider) - self.assertEqual(self.logged(), "[myspider] INFO: Hello") + self.failIf(self.logged()) def test_msg_level1(self): log.msg("Hello", level=log.WARNING) @@ -57,11 +58,6 @@ class ScrapyFileLogObserverTest(unittest.TestCase): log.msg("Hello", level=9999) self.assertEqual(self.logged(), "[scrapy] NOLEVEL: Hello") - def test_msg_level_spider(self): - spider = Spider("myspider") - log.msg("Hello", spider=spider, level=log.WARNING) - self.assertEqual(self.logged(), "[myspider] WARNING: Hello") - def test_msg_encoding(self): log.msg(u"Price: \xa3100") self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xc2\xa3100") @@ -133,5 +129,41 @@ class Latin1ScrapyFileLogObserverTest(ScrapyFileLogObserverTest): # self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xa3") +class CrawlerScrapyFileLogObserverTest(unittest.TestCase): + + def setUp(self): + self.f = BytesIO() + self.crawler = get_crawler(Spider) + self.spider = self.crawler.spider = self.crawler._create_spider('test') + self.log_observer = log.ScrapyFileLogObserver(self.f, log.INFO, + 'utf-8', self.crawler) + self.log_observer.start() + + def tearDown(self): + self.flushLoggedErrors() + self.log_observer.stop() + + def logged(self): + return self.f.getvalue().strip()[25:] + + def test_msg_basic(self): + log.msg("Hello", spider=self.spider) + self.assertEqual(self.logged(), "[test] INFO: Hello") + + def test_msg_ignore_scrapy_channel(self): + log.msg("Hello") + self.failIf(self.logged()) + + def test_msg_ignore_another_crawler(self): + crawler = get_crawler(Spider) + log.msg("Hello", spider=crawler._create_spider('test')) + self.failIf(self.logged()) + + def test_msg_stats_log(self): + assert self.crawler.stats.get_value('log_count/INFO', 0) == 0 + log.msg("Hello", spider=self.spider) + self.assertEqual(self.crawler.stats.get_value('log_count/INFO'), 1) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 8999e102e..5ce48ebf8 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -8,7 +8,7 @@ from netlib import http_auth from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.utils.test import get_testlog, docrawl +from scrapy.utils.test import get_testlog, get_crawler from scrapy.http import Request from tests.spiders import SimpleSpider, SingleRequestSpider from tests.mockserver import MockServer @@ -49,29 +49,29 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_connect_tunnel(self): - spider = SimpleSpider("https://localhost:8999/status?n=200") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("https://localhost:8999/status?n=200") self._assert_got_response_code(200) @defer.inlineCallbacks def test_https_noconnect(self): os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect' - spider = SimpleSpider("https://localhost:8999/status?n=200") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("https://localhost:8999/status?n=200") self._assert_got_response_code(200) os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_connect_tunnel_error(self): - spider = SimpleSpider("https://localhost:99999/status?n=200") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("https://localhost:99999/status?n=200") self._assert_got_tunnel_error() @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888' - spider = SimpleSpider("https://localhost:8999/status?n=200") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("https://localhost:8999/status?n=200") # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. self._assert_got_tunnel_error() @@ -80,17 +80,17 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): request = Request("https://localhost:8999/echo") - spider = SingleRequestSpider(seed=request) - yield docrawl(spider) + crawler = get_crawler(SingleRequestSpider) + yield crawler.crawl(seed=request) self._assert_got_response_code(200) - echo = json.loads(spider.meta['responses'][0].body) + echo = json.loads(crawler.spider.meta['responses'][0].body) self.assertTrue('Proxy-Authorization' not in echo['headers']) @defer.inlineCallbacks def test_https_noconnect_auth_error(self): os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect' - spider = SimpleSpider("https://localhost:8999/status?n=200") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("https://localhost:8999/status?n=200") self._assert_got_response_code(407) def _assert_got_response_code(self, code): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 39e47dec6..38797ad45 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -190,6 +190,42 @@ class SettingsTest(unittest.TestCase): self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5}) self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1') + def test_copy(self): + values = { + 'TEST_BOOL': True, + 'TEST_LIST': ['one', 'two'], + 'TEST_LIST_OF_LISTS': [['first_one', 'first_two'], + ['second_one', 'second_two']] + } + self.settings.setdict(values) + copy = self.settings.copy() + self.settings.set('TEST_BOOL', False) + self.assertTrue(copy.get('TEST_BOOL')) + + test_list = self.settings.get('TEST_LIST') + test_list.append('three') + self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two']) + + test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS') + test_list_of_lists[0].append('first_three') + self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0], + ['first_one', 'first_two']) + + def test_freeze(self): + self.settings.freeze() + with self.assertRaises(TypeError) as cm: + self.settings.set('TEST_BOOL', False) + self.assertEqual(str(cm.exception), + "Trying to modify an immutable Settings object") + + def test_frozencopy(self): + with mock.patch.object(self.settings, 'copy') as mock_copy: + with mock.patch.object(mock_copy, 'freeze') as mock_freeze: + mock_object = self.settings.frozencopy() + mock_copy.assert_call_once() + mock_freeze.assert_call_once() + self.assertEqual(mock_object, mock_copy.return_value) + def test_deprecated_attribute_overrides(self): self.settings.set('BAR', 'fuz', priority='cmdline') with warnings.catch_warnings(record=True) as w: diff --git a/tests/test_spider.py b/tests/test_spider.py index 903eff7b1..148a872dd 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,11 +1,14 @@ import gzip import inspect import warnings -from scrapy.utils.trackref import object_ref from io import BytesIO - from twisted.trial import unittest +try: + from unittest import mock +except ImportError: + import mock +from scrapy import signals from scrapy.spider import Spider, BaseSpider from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse from scrapy.contrib.spiders.init import InitSpider @@ -13,6 +16,8 @@ from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider from scrapy.contrib.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.trackref import object_ref +from scrapy.utils.test import get_crawler class SpiderTest(unittest.TestCase): @@ -46,6 +51,47 @@ class SpiderTest(unittest.TestCase): self.assertRaises(ValueError, self.spider_class) self.assertRaises(ValueError, self.spider_class, somearg='foo') + def test_deprecated_set_crawler_method(self): + spider = self.spider_class('example.com') + crawler = get_crawler() + with warnings.catch_warnings(record=True) as w: + spider.set_crawler(crawler) + self.assertIn("set_crawler", str(w[0].message)) + self.assertTrue(hasattr(spider, 'crawler')) + self.assertIs(spider.crawler, crawler) + self.assertTrue(hasattr(spider, 'settings')) + self.assertIs(spider.settings, crawler.settings) + + def test_from_crawler_crawler_and_settings_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, 'example.com') + self.assertTrue(hasattr(spider, 'crawler')) + self.assertIs(spider.crawler, crawler) + self.assertTrue(hasattr(spider, 'settings')) + self.assertIs(spider.settings, crawler.settings) + + def test_from_crawler_init_call(self): + with mock.patch.object(self.spider_class, '__init__', + return_value=None) as mock_init: + self.spider_class.from_crawler(get_crawler(), 'example.com', + foo='bar') + mock_init.assert_called_once_with('example.com', foo='bar') + + def test_closed_signal_call(self): + class TestSpider(self.spider_class): + closed_called = False + + def closed(self, reason): + self.closed_called = True + + crawler = get_crawler() + spider = TestSpider.from_crawler(crawler, 'example.com') + crawler.signals.send_catch_log(signal=signals.spider_opened, + spider=spider) + crawler.signals.send_catch_log(signal=signals.spider_closed, + spider=spider, reason=None) + self.assertTrue(spider.closed_called) + class InitSpiderTest(SpiderTest): @@ -190,6 +236,32 @@ class CrawlSpiderTest(SpiderTest): 'http://example.org/about.html', 'http://example.org/nofollow.html']) + def test_follow_links_attribute_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, 'example.com') + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertTrue(spider._follow_links) + + settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False} + crawler = get_crawler(settings_dict=settings_dict) + spider = self.spider_class.from_crawler(crawler, 'example.com') + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertFalse(spider._follow_links) + + def test_follow_links_attribute_deprecated_population(self): + spider = self.spider_class('example.com') + self.assertFalse(hasattr(spider, '_follow_links')) + + spider.set_crawler(get_crawler()) + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertTrue(spider._follow_links) + + spider = self.spider_class('example.com') + settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False} + spider.set_crawler(get_crawler(settings_dict=settings_dict)) + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertFalse(spider._follow_links) + class SitemapSpiderTest(SpiderTest): diff --git a/tests/test_spidermanager/__init__.py b/tests/test_spidermanager/__init__.py index b0dd9a851..69ab3b82a 100644 --- a/tests/test_spidermanager/__init__.py +++ b/tests/test_spidermanager/__init__.py @@ -10,6 +10,7 @@ from twisted.trial import unittest # alone from scrapy.interfaces import ISpiderManager from scrapy.spidermanager import SpiderManager +from scrapy.settings import Settings from scrapy.http import Request module_dir = os.path.dirname(os.path.abspath(__file__)) @@ -23,7 +24,8 @@ class SpiderManagerTest(unittest.TestCase): self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') shutil.copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) - self.spiderman = SpiderManager(['test_spiders_xxx']) + settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) + self.spiderman = SpiderManager.from_settings(settings) def tearDown(self): del self.spiderman @@ -35,14 +37,11 @@ class SpiderManagerTest(unittest.TestCase): def test_list(self): self.assertEqual(set(self.spiderman.list()), - set(['spider1', 'spider2', 'spider3', 'spider4'])) + set(['spider1', 'spider2', 'spider3'])) - def test_create(self): - spider1 = self.spiderman.create("spider1") - self.assertEqual(spider1.__class__.__name__, 'Spider1') - spider2 = self.spiderman.create("spider2", foo="bar") - self.assertEqual(spider2.__class__.__name__, 'Spider2') - self.assertEqual(spider2.foo, 'bar') + def test_load(self): + spider1 = self.spiderman.load("spider1") + self.assertEqual(spider1.__name__, 'Spider1') def test_find_by_request(self): self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')), @@ -59,13 +58,13 @@ class SpiderManagerTest(unittest.TestCase): ['spider3']) def test_load_spider_module(self): - self.spiderman = SpiderManager(['tests.test_spidermanager.test_spiders.spider1']) + module = 'tests.test_spidermanager.test_spiders.spider1' + settings = Settings({'SPIDER_MODULES': [module]}) + self.spiderman = SpiderManager.from_settings(settings) assert len(self.spiderman._spiders) == 1 def test_load_base_spider(self): - self.spiderman = SpiderManager(['tests.test_spidermanager.test_spiders.spider0']) + module = 'tests.test_spidermanager.test_spiders.spider0' + settings = Settings({'SPIDER_MODULES': [module]}) + self.spiderman = SpiderManager.from_settings(settings) assert len(self.spiderman._spiders) == 0 - - def test_load_from_crawler(self): - spider = self.spiderman.create('spider4', a='OK') - self.assertEqual(spider.a, 'OK') diff --git a/tests/test_spidermanager/test_spiders/spider4.py b/tests/test_spidermanager/test_spiders/spider4.py deleted file mode 100644 index e883e4d93..000000000 --- a/tests/test_spidermanager/test_spiders/spider4.py +++ /dev/null @@ -1,10 +0,0 @@ -from scrapy.spider import Spider - -class Spider4(Spider): - name = "spider4" - - @classmethod - def from_crawler(cls, crawler, **kwargs): - o = cls(**kwargs) - o.crawler = crawler - return o diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 94404ff41..e7ae75ed2 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -10,9 +10,10 @@ from scrapy.utils.test import get_crawler class TestDepthMiddleware(TestCase): def setUp(self): - self.spider = Spider('scrapytest.org') + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('scrapytest.org') - self.stats = StatsCollector(get_crawler()) + self.stats = StatsCollector(crawler) self.stats.open_spider(self.spider) self.mw = DepthMiddleware(1, self.stats, True) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 788a0986b..5394f0eee 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -3,7 +3,7 @@ from unittest import TestCase from twisted.trial.unittest import TestCase as TrialTestCase from twisted.internet import defer -from scrapy.utils.test import docrawl, get_testlog +from scrapy.utils.test import get_crawler, get_testlog from tests.mockserver import MockServer from scrapy.http import Response, Request from scrapy.spider import Spider @@ -165,20 +165,20 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): @defer.inlineCallbacks def test_middleware_works(self): - spider = _HttpErrorSpider() - yield docrawl(spider) - assert not spider.skipped, spider.skipped - self.assertEqual(spider.parsed, {'200'}) - self.assertEqual(spider.failed, {'404', '402', '500'}) + crawler = get_crawler(_HttpErrorSpider) + yield crawler.crawl() + assert not crawler.spider.skipped, crawler.spider.skipped + self.assertEqual(crawler.spider.parsed, {'200'}) + self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @defer.inlineCallbacks def test_logging(self): - spider = _HttpErrorSpider(bypass_status_codes={402}) - yield docrawl(spider) + crawler = get_crawler(_HttpErrorSpider) + yield crawler.crawl(bypass_status_codes={402}) # print(get_testlog()) - self.assertEqual(spider.parsed, {'200', '402'}) - self.assertEqual(spider.skipped, {'402'}) - self.assertEqual(spider.failed, {'404', '500'}) + self.assertEqual(crawler.spider.parsed, {'200', '402'}) + self.assertEqual(crawler.spider.skipped, {'402'}) + self.assertEqual(crawler.spider.failed, {'404', '500'}) log = get_testlog() self.assertIn('Ignoring response <404', log) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 298cba6e4..e5e99002a 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -10,13 +10,13 @@ from scrapy.utils.test import get_crawler class TestOffsiteMiddleware(TestCase): def setUp(self): - self.spider = self._get_spider() - crawler = get_crawler() + crawler = get_crawler(Spider) + self.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) self.mw.spider_opened(self.spider) - def _get_spider(self): - return Spider('foo', allowed_domains=['scrapytest.org', 'scrapy.org']) + def _get_spiderargs(self): + return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org']) def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -39,8 +39,8 @@ class TestOffsiteMiddleware(TestCase): class TestOffsiteMiddleware2(TestOffsiteMiddleware): - def _get_spider(self): - return Spider('foo', allowed_domains=None) + def _get_spiderargs(self): + return dict(name='foo', allowed_domains=None) def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -58,7 +58,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): def _get_spider(self): bad_hostname = urlparse('http:////scrapytest.org').hostname - return Spider('foo', allowed_domains=['scrapytest.org', None, bad_hostname]) + return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname]) def test_process_spider_output(self): res = Response('http://scrapytest.org') diff --git a/tests/test_stats.py b/tests/test_stats.py index 795e8e3bd..db1f50712 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -7,8 +7,8 @@ from scrapy.utils.test import get_crawler class StatsCollectorTest(unittest.TestCase): def setUp(self): - self.crawler = get_crawler() - self.spider = Spider('foo') + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('foo') def test_collector(self): stats = StatsCollector(self.crawler)