From 84fa004793cb1be07c7a3d0ac6fd80a83b4e8487 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 30 Jun 2014 01:35:58 -0300 Subject: [PATCH 01/18] Add from_crawler class method to base Spider --- docs/topics/spiders.rst | 38 ++++++++++++++++++++++++++++++++++++++ scrapy/spider.py | 29 +++++++++++++++++++---------- tests/test_spider.py | 34 ++++++++++++++++++++++++++++++++-- 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 73c34e75f..de8f988c0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -133,6 +133,44 @@ Spider listed here. The subsequent URLs will be generated successively from data contained in the start URLs. + .. attribute:: crawler + + This attribute is set by the :meth:`from_crawler` class method after + initializating the class, and links to the + :class:`~scrapy.crawler.Crawler` object to which this spider instance is + bound. + + Crawlers encapsulate a lot of components in the project for their single + entry access (such as extensions, middlewares, signals managers, etc). + See :ref:`topics-api-crawler` to know more about them. + + .. attribute:: settings + + Configuration on which this spider is been ran. This is a + :class:`~scrapy.settings.Settings` instance, see the + :ref:`topics-settings` topic for a detailed introduction on this subject. + + .. method:: from_crawler(crawler, \*args, \**kwargs) + + This is the class method used by Scrapy to create your spiders. + + You probably won't need to override this directly, since the default + implementation acts as a proxy to the :meth:`__init__` method, calling + it with the given arguments `args` and named arguments `kwargs`. + + Nonetheless, this method sets the :attr:`crawler` and :attr:`settings` + attributes in the new instance, so they can be accessed later inside the + spider's code. + + :param crawler: crawler to which the spider will be bound + :type crawler: :class:`~scrapy.crawler.Crawler` instance + + :param args: arguments passed to the :meth:`__init__` method + :type args: list + + :param kwargs: keyword arguments passed to the :meth:`__init__` method + :type kwargs: dict + .. method:: start_requests() This method must return an iterable with the first Requests to crawl for diff --git a/scrapy/spider.py b/scrapy/spider.py index 8ecfae2a0..89f78d6ba 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -3,11 +3,14 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ +import warnings + from scrapy import log from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider from scrapy.utils.deprecate import create_deprecated_class +from scrapy.exceptions import ScrapyDeprecationWarning class Spider(object_ref): @@ -32,18 +35,24 @@ class Spider(object_ref): """ log.msg(message, spider=self, level=level, **kw) + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(*args, **kwargs) + spider._set_crawler(crawler) + return spider + def set_crawler(self, crawler): - assert not hasattr(self, '_crawler'), "Spider already bounded to %s" % crawler - self._crawler = crawler + warnings.warn("set_crawler is deprecated, instantiate and bound the " + "spider to this crawler with from_crawler method " + "instead.", + category=ScrapyDeprecationWarning, stacklevel=2) + assert not hasattr(self, 'crawler'), "Spider already bounded to a " \ + "crawler" + self._set_crawler(crawler) - @property - def crawler(self): - assert hasattr(self, '_crawler'), "Spider not bounded to any crawler" - return self._crawler - - @property - def settings(self): - return self.crawler.settings + def _set_crawler(self, crawler): + self.crawler = crawler + self.settings = crawler.settings def start_requests(self): for url in self.start_urls: diff --git a/tests/test_spider.py b/tests/test_spider.py index 903eff7b1..53daf39fb 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,10 +1,12 @@ import gzip import inspect import warnings -from scrapy.utils.trackref import object_ref from io import BytesIO - from twisted.trial import unittest +try: + from unittest import mock +except ImportError: + import mock from scrapy.spider import Spider, BaseSpider from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse @@ -13,6 +15,8 @@ from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider from scrapy.contrib.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.trackref import object_ref +from scrapy.utils.test import get_crawler class SpiderTest(unittest.TestCase): @@ -46,6 +50,32 @@ class SpiderTest(unittest.TestCase): self.assertRaises(ValueError, self.spider_class) self.assertRaises(ValueError, self.spider_class, somearg='foo') + def test_deprecated_set_crawler_method(self): + spider = self.spider_class('example.com') + crawler = get_crawler() + with warnings.catch_warnings(record=True) as w: + spider.set_crawler(crawler) + self.assertIn("set_crawler", str(w[0].message)) + self.assertTrue(hasattr(spider, 'crawler')) + self.assertIs(spider.crawler, crawler) + self.assertTrue(hasattr(spider, 'settings')) + self.assertIs(spider.settings, crawler.settings) + + def test_from_crawler_crawler_and_settings_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, 'example.com') + self.assertTrue(hasattr(spider, 'crawler')) + self.assertIs(spider.crawler, crawler) + self.assertTrue(hasattr(spider, 'settings')) + self.assertIs(spider.settings, crawler.settings) + + def test_from_crawler_init_call(self): + with mock.patch.object(self.spider_class, '__init__', + return_value=None) as mock_init: + self.spider_class.from_crawler(get_crawler(), 'example.com', + foo='bar') + mock_init.assert_called_once_with('example.com', foo='bar') + class InitSpiderTest(SpiderTest): From eb0253e5301ea54d4f37da8974d0dac295ebe871 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 30 Jun 2014 03:20:05 -0300 Subject: [PATCH 02/18] Update from_crawler method as well as set_crawler on CrawlSpider --- scrapy/contrib/spiders/crawl.py | 7 +++++++ tests/test_spider.py | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index d7e42f6f8..7dc3dacd6 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -86,6 +86,13 @@ class CrawlSpider(Spider): rule.process_links = get_method(rule.process_links) rule.process_request = get_method(rule.process_request) + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs) + spider._follow_links = crawler.settings.getbool( + 'CRAWLSPIDER_FOLLOW_LINKS', True) + return spider + def set_crawler(self, crawler): super(CrawlSpider, self).set_crawler(crawler) self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) diff --git a/tests/test_spider.py b/tests/test_spider.py index 53daf39fb..188bef6e4 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -220,6 +220,30 @@ class CrawlSpiderTest(SpiderTest): 'http://example.org/about.html', 'http://example.org/nofollow.html']) + def test_follow_links_attribute_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, 'example.com') + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertTrue(spider._follow_links) + + crawler.settings.set('CRAWLSPIDER_FOLLOW_LINKS', False) + spider = self.spider_class.from_crawler(crawler, 'example.com') + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertFalse(spider._follow_links) + + def test_follow_links_attribute_deprecated_population(self): + spider = self.spider_class('example.com') + self.assertFalse(hasattr(spider, '_follow_links')) + + spider.set_crawler(get_crawler()) + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertTrue(spider._follow_links) + + spider = self.spider_class('example.com') + spider.set_crawler(get_crawler({'CRAWLSPIDER_FOLLOW_LINKS': False})) + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertFalse(spider._follow_links) + class SitemapSpiderTest(SpiderTest): From a995727117d10133a20553a648e85970fc6a6543 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 17 Jul 2014 10:49:15 -0300 Subject: [PATCH 03/18] Connect spider_closed signal after a crawler is bound to a Spider --- scrapy/spider.py | 8 ++++++++ tests/test_spider.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/scrapy/spider.py b/scrapy/spider.py index 89f78d6ba..df367b700 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -6,6 +6,7 @@ See documentation in docs/topics/spiders.rst import warnings from scrapy import log +from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider @@ -53,6 +54,7 @@ class Spider(object_ref): def _set_crawler(self, crawler): self.crawler = crawler self.settings = crawler.settings + crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self): for url in self.start_urls: @@ -68,6 +70,12 @@ class Spider(object_ref): def handles_request(cls, request): return url_is_from_spider(request.url, cls) + @staticmethod + def close(spider, reason): + closed = getattr(spider, 'closed', None) + if callable(closed): + return closed(reason) + def __str__(self): return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self)) diff --git a/tests/test_spider.py b/tests/test_spider.py index 188bef6e4..903ea684a 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -8,6 +8,7 @@ try: except ImportError: import mock +from scrapy import signals from scrapy.spider import Spider, BaseSpider from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse from scrapy.contrib.spiders.init import InitSpider @@ -76,6 +77,21 @@ class SpiderTest(unittest.TestCase): foo='bar') mock_init.assert_called_once_with('example.com', foo='bar') + def test_closed_signal_call(self): + class TestSpider(self.spider_class): + closed_called = False + + def closed(self, reason): + self.closed_called = True + + crawler = get_crawler() + spider = TestSpider.from_crawler(crawler, 'example.com') + crawler.signals.send_catch_log(signal=signals.spider_opened, + spider=spider) + crawler.signals.send_catch_log(signal=signals.spider_closed, + spider=spider, reason=None) + self.assertTrue(spider.closed_called) + class InitSpiderTest(SpiderTest): From 3ae971468ff3a6712aa47ad3a5d2b9f0c9663b60 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 29 Jul 2014 18:47:49 -0300 Subject: [PATCH 04/18] Add Settings.copy, freeze and frozencopy method --- docs/topics/api.rst | 24 ++++++++++++++++++++++ scrapy/settings/__init__.py | 16 +++++++++++++++ tests/test_settings/__init__.py | 36 +++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 341340c2a..16bfe5f8f 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -264,6 +264,30 @@ Settings API :param default: the value to return if no setting is found :type default: any + .. method:: copy() + + Make a deep copy of current settings. + + This method returns a new instance of the :class:`Settings` class, + populated with the same values and their priorities. + + Modifications to the new object won't be reflected on the original + settings. + + .. method:: freeze() + + Disable further changes to the current settings. + + After calling this method, the present state of the settings will become + immutable. Trying to change values through the :meth:`~set` method and + its variants won't be possible and will be alerted. + + .. method:: frozencopy() + + Return an immutable copy of the current settings. + + Alias for a :meth:`~freeze` call in the object returned by :meth:`copy` + .. _topics-api-signals: Signals API diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 2dd6a2920..978174694 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,5 +1,6 @@ import six import json +import copy import warnings from collections import MutableMapping from importlib import import_module @@ -46,6 +47,7 @@ class SettingsAttribute(object): class Settings(object): def __init__(self, values=None, priority='project'): + self.frozen = False self.attributes = {} self.setmodule(default_settings, priority='default') if values is not None: @@ -93,6 +95,7 @@ class Settings(object): raise ValueError("Cannot convert value for setting '%s' to dict: '%s'" % (name, value)) def set(self, name, value, priority='project'): + assert not self.frozen, "Trying to modify an immutable Settings object" if isinstance(priority, six.string_types): priority = SETTINGS_PRIORITIES[priority] if name not in self.attributes: @@ -101,16 +104,29 @@ class Settings(object): self.attributes[name].set(value, priority) def setdict(self, values, priority='project'): + assert not self.frozen, "Trying to modify an immutable Settings object" for name, value in six.iteritems(values): self.set(name, value, priority) def setmodule(self, module, priority='project'): + assert not self.frozen, "Trying to modify an immutable Settings object" if isinstance(module, six.string_types): module = import_module(module) for key in dir(module): if key.isupper(): self.set(key, getattr(module, key), priority) + def copy(self): + return copy.deepcopy(self) + + def freeze(self): + self.frozen = True + + def frozencopy(self): + copy = self.copy() + copy.freeze() + return copy + @property def overrides(self): warnings.warn("`Settings.overrides` attribute is deprecated and won't " diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 39e47dec6..c7e0914d6 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -190,6 +190,42 @@ class SettingsTest(unittest.TestCase): self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5}) self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1') + def test_copy(self): + values = { + 'TEST_BOOL': True, + 'TEST_LIST': ['one', 'two'], + 'TEST_LIST_OF_LISTS': [['first_one', 'first_two'], + ['second_one', 'second_two']] + } + self.settings.setdict(values) + copy = self.settings.copy() + self.settings.set('TEST_BOOL', False) + self.assertTrue(copy.get('TEST_BOOL')) + + test_list = self.settings.get('TEST_LIST') + test_list.append('three') + self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two']) + + test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS') + test_list_of_lists[0].append('first_three') + self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0], + ['first_one', 'first_two']) + + def test_freeze(self): + self.settings.freeze() + with self.assertRaises(AssertionError) as cm: + self.settings.set('TEST_BOOL', False) + self.assertEqual(str(cm.exception), + "Trying to modify an immutable Settings object") + + def test_frozencopy(self): + with mock.patch.object(self.settings, 'copy') as mock_copy: + with mock.patch.object(mock_copy, 'freeze') as mock_freeze: + mock_object = self.settings.frozencopy() + mock_copy.assert_call_once() + mock_freeze.assert_call_once() + self.assertEqual(mock_object, mock_copy.return_value) + def test_deprecated_attribute_overrides(self): self.settings.set('BAR', 'fuz', priority='cmdline') with warnings.catch_warnings(record=True) as w: From 39c6a80f9db6ec04cac59f116ee9620c3d540be0 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 1 Aug 2014 00:42:25 -0300 Subject: [PATCH 05/18] Both getdict and getlist return copies of the requested values --- docs/topics/api.rst | 16 ++++++++++++++-- scrapy/settings/__init__.py | 19 ++++++------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 16bfe5f8f..6e636e826 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -252,8 +252,8 @@ Settings API .. method:: getlist(name, default=None) - Get a setting value as a list. If the setting original type is a list it - will be returned verbatim. If it's a string it will be split by ",". + Get a setting value as a list. If the setting original type is a list, a + copy of it will be returned. If it's a string it will be split by ",". For example, settings populated through environment variables set to ``'one,two'`` will return a list ['one', 'two'] when using this method. @@ -264,6 +264,18 @@ Settings API :param default: the value to return if no setting is found :type default: any + .. method:: getdict(name, default=None) + + Get a setting value as a dictionary. If the setting original type is a + dictionary, a copy of it will be returned. If it's a string it will + evaluated as a json dictionary. + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + .. method:: copy() Make a deep copy of current settings. diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 978174694..bbe8ef481 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -76,23 +76,16 @@ class Settings(object): return float(self.get(name, default)) def getlist(self, name, default=None): - value = self.get(name) - if value is None: - return default or [] - elif hasattr(value, '__iter__'): - return value - else: - return str(value).split(',') + value = self.get(name, default or []) + if isinstance(value, six.string_types): + value = value.split(',') + return list(value) def getdict(self, name, default=None): - value = self.get(name) - if value is None: - return default or {} + value = self.get(name, default or {}) if isinstance(value, six.string_types): value = json.loads(value) - if isinstance(value, dict): - return value - raise ValueError("Cannot convert value for setting '%s' to dict: '%s'" % (name, value)) + return dict(value) def set(self, name, value, priority='project'): assert not self.frozen, "Trying to modify an immutable Settings object" From d7038b2a136a2b79df9fb16d7b3327cb29f9c46f Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 17 Jul 2014 10:25:07 -0300 Subject: [PATCH 06/18] SpiderManager interface cleanup --- docs/topics/api.rst | 51 ++++++++++++++++++- docs/topics/settings.rst | 10 ++++ scrapy/interfaces.py | 10 ++-- scrapy/spidermanager.py | 31 +++-------- tests/test_spidermanager/__init__.py | 27 +++++----- .../test_spiders/spider4.py | 10 ---- 6 files changed, 85 insertions(+), 54 deletions(-) delete mode 100644 tests/test_spidermanager/test_spiders/spider4.py diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 6e636e826..348305fb7 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -77,8 +77,7 @@ how you :ref:`configure the downloader middlewares .. attribute:: spiders - The spider manager which takes care of loading and instantiating - spiders. + The spider manager which takes care of loading spiders. Most extensions won't need to access this attribute. @@ -300,6 +299,54 @@ Settings API Alias for a :meth:`~freeze` call in the object returned by :meth:`copy` +.. _topics-api-spidermanager: + +SpiderManager API +================= + +.. module:: scrapy.spidermanager + :synopsis: The spider manager + +.. class:: SpiderManager + + This class is in charge of retrieving and handling the spider classes + defined across the project. + + Custom spider managers can be employed by specifying their path in the + :setting:`SPIDER_MANAGER_CLASS` project setting. They must fully implement + the :class:`scrapy.interfaces.ISpiderManager` interface to guarantee an + errorless execution. + + .. method:: from_settings(settings) + + This class method is used by Scrapy to create an instance of the class. + It's called with the current project settings, and it loads the spiders + found in the modules of the :setting:`SPIDER_MODULES` setting. + + :param settings: project settings + :type settings: :class:`~scrapy.settings.Settings` instance + + .. method:: load(spider_name) + + Get the Spider class with the given name. It'll look into the previously + loaded spiders for a spider class with name `spider_name` and will raise + a KeyError if not found. + + :param spider_name: spider class name + :type spider_name: str + + .. method:: list() + + Get the names of the available spiders in the project. + + .. method:: find_by_request(request) + + List the spiders' names that can handle the given request. Will try to + match the request's url against the domains of the spiders. + + :param request: queried request + :type request: :class:`~scrapy.http.Request` instance + .. _topics-api-signals: Signals API diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 629fac2dd..8eb72eaea 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -768,6 +768,16 @@ A dict containing the scrapy contracts enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS` instead. For more info see :ref:`topics-contracts`. +.. setting:: SPIDER_MANAGER_CLASS + +SPIDER_MANAGER_CLASS +-------------------- + +Default: ``'scrapy.spidermanager.SpiderManager'`` + +The class that will be used for handling spiders, which must implement the +:ref:`topics-api-spidermanager`. + .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index 5d8d85aca..d4596407e 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -2,10 +2,12 @@ from zope.interface import Interface class ISpiderManager(Interface): - def create(spider_name, **spider_args): - """Returns a new Spider instance for the given spider name, and using - the given spider arguments. If the spider name is not found, it must - raise a KeyError.""" + def from_settings(settings): + """Returns an instance of the class for the given settings""" + + def load(spider_name): + """Returns the Spider class for the given spider name. If the spider + name is not found, it must raise a KeyError.""" def list(): """Return a list with the names of all spiders available in the diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py index 5a0951cb4..5715b7793 100644 --- a/scrapy/spidermanager.py +++ b/scrapy/spidermanager.py @@ -6,7 +6,6 @@ spiders from zope.interface import implementer import six -from scrapy import signals from scrapy.interfaces import ISpiderManager from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes @@ -15,8 +14,8 @@ from scrapy.utils.spider import iter_spider_classes @implementer(ISpiderManager) class SpiderManager(object): - def __init__(self, spider_modules): - self.spider_modules = spider_modules + def __init__(self, settings): + self.spider_modules = settings['SPIDER_MODULES'] self._spiders = {} for name in self.spider_modules: for module in walk_modules(name): @@ -28,33 +27,17 @@ class SpiderManager(object): @classmethod def from_settings(cls, settings): - return cls(settings.getlist('SPIDER_MODULES')) + return cls(settings) - @classmethod - def from_crawler(cls, crawler): - sm = cls.from_settings(crawler.settings) - sm.crawler = crawler - crawler.signals.connect(sm.close_spider, signals.spider_closed) - return sm - - def create(self, spider_name, **spider_kwargs): + def load(self, spider_name): try: - spcls = self._spiders[spider_name] + return self._spiders[spider_name] except KeyError: - raise KeyError("Spider not found: %s" % spider_name) - if hasattr(self, 'crawler') and hasattr(spcls, 'from_crawler'): - return spcls.from_crawler(self.crawler, **spider_kwargs) - else: - return spcls(**spider_kwargs) + raise KeyError("Spider not found: {}".format(spider_name)) def find_by_request(self, request): return [name for name, cls in six.iteritems(self._spiders) if cls.handles_request(request)] def list(self): - return self._spiders.keys() - - def close_spider(self, spider, reason): - closed = getattr(spider, 'closed', None) - if callable(closed): - return closed(reason) + return list(self._spiders.keys()) diff --git a/tests/test_spidermanager/__init__.py b/tests/test_spidermanager/__init__.py index b0dd9a851..69ab3b82a 100644 --- a/tests/test_spidermanager/__init__.py +++ b/tests/test_spidermanager/__init__.py @@ -10,6 +10,7 @@ from twisted.trial import unittest # alone from scrapy.interfaces import ISpiderManager from scrapy.spidermanager import SpiderManager +from scrapy.settings import Settings from scrapy.http import Request module_dir = os.path.dirname(os.path.abspath(__file__)) @@ -23,7 +24,8 @@ class SpiderManagerTest(unittest.TestCase): self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') shutil.copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) - self.spiderman = SpiderManager(['test_spiders_xxx']) + settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) + self.spiderman = SpiderManager.from_settings(settings) def tearDown(self): del self.spiderman @@ -35,14 +37,11 @@ class SpiderManagerTest(unittest.TestCase): def test_list(self): self.assertEqual(set(self.spiderman.list()), - set(['spider1', 'spider2', 'spider3', 'spider4'])) + set(['spider1', 'spider2', 'spider3'])) - def test_create(self): - spider1 = self.spiderman.create("spider1") - self.assertEqual(spider1.__class__.__name__, 'Spider1') - spider2 = self.spiderman.create("spider2", foo="bar") - self.assertEqual(spider2.__class__.__name__, 'Spider2') - self.assertEqual(spider2.foo, 'bar') + def test_load(self): + spider1 = self.spiderman.load("spider1") + self.assertEqual(spider1.__name__, 'Spider1') def test_find_by_request(self): self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')), @@ -59,13 +58,13 @@ class SpiderManagerTest(unittest.TestCase): ['spider3']) def test_load_spider_module(self): - self.spiderman = SpiderManager(['tests.test_spidermanager.test_spiders.spider1']) + module = 'tests.test_spidermanager.test_spiders.spider1' + settings = Settings({'SPIDER_MODULES': [module]}) + self.spiderman = SpiderManager.from_settings(settings) assert len(self.spiderman._spiders) == 1 def test_load_base_spider(self): - self.spiderman = SpiderManager(['tests.test_spidermanager.test_spiders.spider0']) + module = 'tests.test_spidermanager.test_spiders.spider0' + settings = Settings({'SPIDER_MODULES': [module]}) + self.spiderman = SpiderManager.from_settings(settings) assert len(self.spiderman._spiders) == 0 - - def test_load_from_crawler(self): - spider = self.spiderman.create('spider4', a='OK') - self.assertEqual(spider.a, 'OK') diff --git a/tests/test_spidermanager/test_spiders/spider4.py b/tests/test_spidermanager/test_spiders/spider4.py deleted file mode 100644 index e883e4d93..000000000 --- a/tests/test_spidermanager/test_spiders/spider4.py +++ /dev/null @@ -1,10 +0,0 @@ -from scrapy.spider import Spider - -class Spider4(Spider): - name = "spider4" - - @classmethod - def from_crawler(cls, crawler, **kwargs): - o = cls(**kwargs) - o.crawler = crawler - return o From 980e30a18758f21f416036c2076bf98630c70193 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 29 Jul 2014 17:46:42 -0300 Subject: [PATCH 07/18] Crawler interface cleanup --- docs/topics/api.rst | 17 ++++++----- scrapy/crawler.py | 70 ++++++++++++++++++++++----------------------- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 348305fb7..229943c55 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -28,9 +28,10 @@ contains a dictionary of all available extensions and their order similar to how you :ref:`configure the downloader middlewares `. -.. class:: Crawler(settings) +.. class:: Crawler(spidercls, settings) The Crawler object must be instantiated with a + :class:`scrapy.spider.Spider` subclass and a :class:`scrapy.settings.Settings` object. .. attribute:: settings @@ -90,16 +91,18 @@ how you :ref:`configure the downloader middlewares or modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. - .. method:: configure() + .. attribute:: spider - Configure the crawler. + Spider currently being crawled. This is an instance of the spider class + provided while constructing the crawler, and it is created after the + arguments given in the :meth:`crawl` method. - This loads extensions, middlewares and spiders, leaving the crawler - ready to be started. It also configures the execution engine. + .. method:: crawl(\*args, \**kwargs) - .. method:: start() + Starts the crawler by instantiating its spider class with the given + `args` and `kwargs` arguments, while setting the execution engine in + motion. - Start the crawler. This calls :meth:`configure` if it hasn't been called yet. Returns a deferred that is fired when the crawl is finished. .. _topics-api-settings: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index cfd6c8003..db1a083dd 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -13,16 +13,22 @@ from scrapy import log, signals class Crawler(object): - def __init__(self, settings): - self.configured = False + def __init__(self, spidercls, settings): + self.spidercls = spidercls self.settings = settings self.signals = SignalManager(self) - self.stats = load_object(settings['STATS_CLASS'])(self) - self._start_requests = lambda: () - self._spider = None - # TODO: move SpiderManager to CrawlerProcess + self.stats = load_object(self.settings['STATS_CLASS'])(self) + lf_cls = load_object(self.settings['LOG_FORMATTER']) + self.logformatter = lf_cls.from_crawler(self) + self.extensions = ExtensionManager.from_crawler(self) + + # Attribute kept for backward compatibility (Use CrawlerRunner.spiders) spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) - self.spiders = spman_cls.from_crawler(self) + self.spiders = spman_cls.from_settings(self.settings) + + self.crawling = False + self.spider = None + self.engine = None def install(self): # TODO: remove together with scrapy.project.crawler usage @@ -36,39 +42,31 @@ class Crawler(object): assert hasattr(scrapy.project, 'crawler'), "crawler not installed" del scrapy.project.crawler - def configure(self): - if self.configured: - return - - self.configured = True - lf_cls = load_object(self.settings['LOG_FORMATTER']) - self.logformatter = lf_cls.from_crawler(self) - self.extensions = ExtensionManager.from_crawler(self) - self.engine = ExecutionEngine(self, self._spider_closed) - - def crawl(self, spider, requests=None): - assert self._spider is None, 'Spider already attached' - self._spider = spider - spider.set_crawler(self) - if requests is None: - self._start_requests = spider.start_requests - else: - self._start_requests = lambda: requests - - def _spider_closed(self, spider=None): - if not self.engine.open_spiders: - self.stop() - @defer.inlineCallbacks - def start(self): - yield defer.maybeDeferred(self.configure) - if self._spider: - yield self.engine.open_spider(self._spider, self._start_requests()) - yield defer.maybeDeferred(self.engine.start) + def crawl(self, *args, **kwargs): + assert not self.crawling, "Crawling already taking place" + self.crawling = True + + try: + self.spider = self._create_spider(*args, **kwargs) + self.engine = self._create_engine() + start_requests = iter(self.spider.start_requests()) + yield self.engine.open_spider(self.spider, start_requests) + yield defer.maybeDeferred(self.engine.start) + except Exception: + self.crawling = False + raise + + def _create_spider(self, *args, **kwargs): + return self.spidercls.from_crawler(self, *args, **kwargs) + + def _create_engine(self): + return ExecutionEngine(self, lambda _: self.stop()) @defer.inlineCallbacks def stop(self): - if self.configured and self.engine.running: + if self.crawling: + self.crawling = False yield defer.maybeDeferred(self.engine.stop) From d40273561dad76a409f847b5f8ce1daafdb1dc7c Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Wed, 30 Jul 2014 05:35:18 -0300 Subject: [PATCH 08/18] CrawlerProcess cleanup changes --- docs/topics/api.rst | 48 +++++++++++++++++++ docs/topics/practices.rst | 76 ++++++++++++++++++------------ scrapy/crawler.py | 98 ++++++++++++++++++++------------------- 3 files changed, 144 insertions(+), 78 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 229943c55..0329e2a8f 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -105,6 +105,53 @@ how you :ref:`configure the downloader middlewares Returns a deferred that is fired when the crawl is finished. +.. class:: CrawlerRunner(settings) + + This is a convenient helper class that creates, configures and runs + crawlers inside an already setup Twisted `reactor`_. + + The CrawlerRunner object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + .. attribute:: crawlers + + Set of :class:`crawlers ` created by the + :meth:`crawl` method. + + .. attribute:: crawl_deferreds + + Set of the `deferreds`_ return by the :meth:`crawl` method. This + collection it's useful for keeping track of current crawling state. + + .. method:: crawl(spidercls, \*args, \**kwargs) + + This method sets up the crawling of the given `spidercls` with the + provided arguments. + + It takes care of loading the spider class while configuring and starting + a crawler for it. + + Returns a deferred that is fired when the crawl is finished. + + :param spidercls: spider class or spider's name inside the project + :type spidercls: :class:`~scrapy.spider.Spider` subclass or str + + :param args: arguments to initializate the spider + :type args: list + + :param kwargs: keyword arguments to initializate the spider + :type kwargs: dict + + .. method:: stop() + + Stops simultaneously all the crawling jobs taking place. + + Returns a deferred that is fired when they all have ended. + .. _topics-api-settings: Settings API @@ -470,3 +517,4 @@ class (which they all inherit from). .. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html .. _deferred: http://twistedmatrix.com/documents/current/core/howto/defer.html +.. _reactor: http://twistedmatrix.com/documents/current/core/howto/reactor-basics.html diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 64b3a2da7..e84478d3c 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -19,8 +19,9 @@ Remember that Scrapy is built on top of the Twisted asynchronous networking library, so you need to run it inside the Twisted reactor. Note that you will also have to shutdown the Twisted reactor yourself after the -spider is finished. This can be achieved by connecting a handler to the -``signals.spider_closed`` signal. +spider is finished. This can be achieved by adding callbacks to the deferred +returned by the :meth:`CrawlerRunner.crawl +` method. What follows is a working example of how to do that, using the `testspiders`_ project as example. @@ -28,20 +29,21 @@ project as example. :: from twisted.internet import reactor - from scrapy.crawler import Crawler - from scrapy import log, signals - from testspiders.spiders.followall import FollowAllSpider + from scrapy.crawler import CrawlerRunner from scrapy.utils.project import get_project_settings - spider = FollowAllSpider(domain='scrapinghub.com') - settings = get_project_settings() - crawler = Crawler(settings) - crawler.signals.connect(reactor.stop, signal=signals.spider_closed) - crawler.configure() - crawler.crawl(spider) - crawler.start() - log.start() - reactor.run() # the script will block here until the spider_closed signal was sent + # If you aren't inside a Scrapy project, you could use an instance of the + # Settings class in scrapy.settings instead of the configuration returned + # by get_project_settings + runner = CrawlerRunner(get_project_settings()) + + # 'followall' is the name of one of the spiders of the project. If you + # aren't working in a Scrapy project, use the spider class as first + # argument instead of its name (or set the SPIDER_MODULES setting so Scrapy + # knows where to look at) + d = runner.crawl('followall', domain='scrapinghub.com') + d.addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until the crawling is finished .. seealso:: `Twisted Reactor Overview`_. @@ -52,28 +54,42 @@ By default, Scrapy runs a single spider per process when you run ``scrapy crawl``. However, Scrapy supports running multiple spiders per process using the :ref:`internal API `. -Here is an example, using the `testspiders`_ project: +Here is an example that runs multiple spiders simultaneously, using the +`testspiders`_ project: :: - from twisted.internet import reactor - from scrapy.crawler import Crawler - from scrapy import log - from testspiders.spiders.followall import FollowAllSpider + from twisted.internet import reactor, defer + from scrapy.crawler import CrawlerRunner from scrapy.utils.project import get_project_settings - def setup_crawler(domain): - spider = FollowAllSpider(domain=domain) - settings = get_project_settings() - crawler = Crawler(settings) - crawler.configure() - crawler.crawl(spider) - crawler.start() - + runner = CrawlerRunner(get_project_settings()) + dfs = set() for domain in ['scrapinghub.com', 'insophia.com']: - setup_crawler(domain) - log.start() - reactor.run() + d = runner.crawl('followall', domain=domain) + dfs.add(d) + + defer.DeferredList(dfs).addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until all crawling jobs are finished + +Same example but running the spiders sequentially by chaining the deferreds: + +:: + + from twisted.internet import reactor, defer + from scrapy.crawler import CrawlerRunner + from scrapy.utils.project import get_project_settings + + runner = CrawlerRunner(get_project_settings()) + + @defer.inlineCallbacks + def crawl(): + for domain in ['scrapinghub.com', 'insophia.com']: + yield runner.crawl('followall', domain=domain) + reactor.stop() + + crawl() + reactor.run() # the script will block here until the last crawl call is finished .. seealso:: :ref:`run-from-script`. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index db1a083dd..56823166b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,3 +1,4 @@ +import six import signal from twisted.internet import reactor, defer @@ -70,31 +71,50 @@ class Crawler(object): yield defer.maybeDeferred(self.engine.stop) -class CrawlerProcess(object): - """ A class to run multiple scrapy crawlers in a process sequentially""" +class CrawlerRunner(object): def __init__(self, settings): - install_shutdown_handlers(self._signal_shutdown) self.settings = settings - self.crawlers = {} - self.stopping = False - self._started = None + smcls = load_object(settings['SPIDER_MANAGER_CLASS']) + self.spiders = smcls.from_settings(settings.frozencopy()) + self.crawlers = set() + self.crawl_deferreds = set() - def create_crawler(self, name=None): - if name not in self.crawlers: - self.crawlers[name] = Crawler(self.settings) + def crawl(self, spidercls, *args, **kwargs): + crawler = self._create_logged_crawler(spidercls) + self.crawlers.add(crawler) - return self.crawlers[name] + crawler.install() + crawler.signals.connect(crawler.uninstall, signals.engine_stopped) - def start(self): - if self.start_crawling(): - self.start_reactor() + d = crawler.crawl(*args, **kwargs) + self.crawl_deferreds.add(d) + return d + + def _create_logged_crawler(self, spidercls): + crawler = self._create_crawler(spidercls) + log_observer = log.start_from_crawler(crawler) + if log_observer: + crawler.signals.connect(log_observer.stop, signals.engine_stopped) + return crawler + + def _create_crawler(self, spidercls): + if isinstance(spidercls, six.string_types): + spidercls = self.spiders.load(spidercls) + crawler = Crawler(spidercls, self.settings.frozencopy()) + return crawler - @defer.inlineCallbacks def stop(self): - self.stopping = True - if self._active_crawler: - yield self._active_crawler.stop() + return defer.DeferredList(c.stop() for c in self.crawlers) + + +class CrawlerProcess(CrawlerRunner): + """A class to run multiple scrapy crawlers in a process simultaneously""" + + def __init__(self, settings): + super(CrawlerProcess, self).__init__(settings) + install_shutdown_handlers(self._signal_shutdown) + self.stopping = False def _signal_shutdown(self, signum, _): install_shutdown_handlers(self._signal_kill) @@ -110,43 +130,25 @@ class CrawlerProcess(object): level=log.INFO, signame=signame) reactor.callFromThread(self._stop_reactor) - # ------------------------------------------------------------------------# - # The following public methods can't be considered stable and may change at - # any moment. - # - # start_crawling and start_reactor are called from scrapy.commands.shell - # They are splitted because reactor is started on a different thread than IPython shell. - # - def start_crawling(self): - log.scrapy_info(self.settings) - return self._start_crawler() is not None + def start(self, stop_after_crawl=True): + self._start_logging() + self._start_reactor(stop_after_crawl) - def start_reactor(self): + def _start_logging(self): + log.scrapy_info(self.settings) + + def _start_reactor(self, stop_after_crawl=True): + if stop_after_crawl: + d = defer.DeferredList(self.crawl_deferreds) + if d.called: + # Don't start the reactor if the deferreds are already fired + return + d.addBoth(lambda _: self._stop_reactor()) if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call - def _start_crawler(self): - if not self.crawlers or self.stopping: - return - - name, crawler = self.crawlers.popitem() - self._active_crawler = crawler - log_observer = log.start_from_crawler(crawler) - crawler.configure() - crawler.install() - crawler.signals.connect(crawler.uninstall, signals.engine_stopped) - if log_observer: - crawler.signals.connect(log_observer.stop, signals.engine_stopped) - crawler.signals.connect(self._check_done, signals.engine_stopped) - crawler.start() - return name, crawler - - def _check_done(self, **kwargs): - if not self._start_crawler(): - self._stop_reactor() - def _stop_reactor(self, _=None): try: reactor.stop() From 870438e5f4665de836f7ff423055895b305a4e7f Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 31 Jul 2014 04:12:12 -0300 Subject: [PATCH 09/18] Update tests utils, fixing get_crawler and removing docrawl --- scrapy/utils/test.py | 15 +- tests/py3-ignores.txt | 5 + tests/test_closespider.py | 38 +++--- tests/test_crawl.py | 128 +++++++++--------- tests/test_downloader_handlers.py | 9 +- tests/test_downloadermiddleware.py | 5 +- ...test_downloadermiddleware_ajaxcrawlable.py | 4 +- ...est_downloadermiddleware_defaultheaders.py | 5 +- ...st_downloadermiddleware_downloadtimeout.py | 5 +- tests/test_downloadermiddleware_httpcache.py | 4 +- tests/test_downloadermiddleware_redirect.py | 8 +- tests/test_downloadermiddleware_retry.py | 4 +- tests/test_downloadermiddleware_stats.py | 4 +- tests/test_downloadermiddleware_useragent.py | 5 +- tests/test_engine.py | 8 +- tests/test_proxy_connect.py | 28 ++-- tests/test_spider.py | 6 +- tests/test_spidermiddleware_depth.py | 5 +- tests/test_spidermiddleware_httperror.py | 22 +-- tests/test_spidermiddleware_offsite.py | 14 +- tests/test_stats.py | 4 +- 21 files changed, 160 insertions(+), 166 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index e6376d519..a4b769970 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -20,15 +20,17 @@ def assert_aws_environ(): if 'AWS_ACCESS_KEY_ID' not in os.environ: raise SkipTest("AWS keys not found") -def get_crawler(settings_dict=None): +def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. """ - from scrapy.crawler import Crawler + from scrapy.crawler import CrawlerRunner from scrapy.settings import Settings + from scrapy.spider import Spider - return Crawler(Settings(settings_dict)) + runner = CrawlerRunner(Settings(settings_dict)) + return runner._create_crawler(spidercls or Spider) def get_pythonpath(): """Return a PYTHONPATH suitable to use in processes so that they find this @@ -62,10 +64,3 @@ def assert_samelines(testcase, text1, text2, msg=None): line endings between platforms """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) - -def docrawl(spider, settings=None): - """Configure and start Crawler; return the result of crawler.start()""" - crawler = get_crawler(settings) - crawler.configure() - crawler.crawl(spider) - return crawler.start() diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index c1619b3ae..ef88eab7e 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -14,6 +14,7 @@ tests/test_downloadermiddleware_ajaxcrawlable.py tests/test_downloadermiddleware_cookies.py tests/test_downloadermiddleware_decompression.py tests/test_downloadermiddleware_defaultheaders.py +tests/test_downloadermiddleware_downloadtimeout.py tests/test_downloadermiddleware_httpauth.py tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py @@ -22,6 +23,7 @@ tests/test_downloadermiddleware.py tests/test_downloadermiddleware_redirect.py tests/test_downloadermiddleware_retry.py tests/test_downloadermiddleware_robotstxt.py +tests/test_downloadermiddleware_stats.py tests/test_downloadermiddleware_useragent.py tests/test_dupefilter.py tests/test_engine.py @@ -48,9 +50,12 @@ tests/test_spidermanager/test_spiders/spider1.py tests/test_spidermanager/test_spiders/spider2.py tests/test_spidermanager/test_spiders/spider3.py tests/test_spidermanager/test_spiders/spider4.py +tests/test_spidermiddleware_depth.py tests/test_spidermiddleware_httperror.py +tests/test_spidermiddleware_offsite.py tests/test_spidermiddleware_referer.py tests/test_spider.py +tests/test_stats.py tests/test_utils_defer.py tests/test_utils_iterators.py tests/test_utils_jsonrpc.py diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 8d30a4643..1700a861e 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,6 +1,6 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.utils.test import docrawl +from scrapy.utils.test import get_crawler from tests.spiders import FollowAllSpider, ItemSpider, ErrorSpider from tests.mockserver import MockServer @@ -16,45 +16,45 @@ class TestCloseSpider(TestCase): @defer.inlineCallbacks def test_closespider_itemcount(self): - spider = ItemSpider() close_on = 5 - yield docrawl(spider, {'CLOSESPIDER_ITEMCOUNT': close_on}) - reason = spider.meta['close_reason'] + crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on}) + yield crawler.crawl() + reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_itemcount') - itemcount = spider.crawler.stats.get_value('item_scraped_count') + itemcount = crawler.stats.get_value('item_scraped_count') self.assertTrue(itemcount >= close_on) @defer.inlineCallbacks def test_closespider_pagecount(self): - spider = FollowAllSpider() close_on = 5 - yield docrawl(spider, {'CLOSESPIDER_PAGECOUNT': close_on}) - reason = spider.meta['close_reason'] + crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on}) + yield crawler.crawl() + reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_pagecount') - pagecount = spider.crawler.stats.get_value('response_received_count') + pagecount = crawler.stats.get_value('response_received_count') self.assertTrue(pagecount >= close_on) @defer.inlineCallbacks def test_closespider_errorcount(self): - spider = ErrorSpider(total=1000000) close_on = 5 - yield docrawl(spider, {'CLOSESPIDER_ERRORCOUNT': close_on}) - self.flushLoggedErrors(spider.exception_cls) - reason = spider.meta['close_reason'] + crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on}) + yield crawler.crawl(total=1000000) + self.flushLoggedErrors(crawler.spider.exception_cls) + reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') key = 'spider_exceptions/{name}'\ - .format(name=spider.exception_cls.__name__) - errorcount = spider.crawler.stats.get_value(key) + .format(name=crawler.spider.exception_cls.__name__) + errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) @defer.inlineCallbacks def test_closespider_timeout(self): - spider = FollowAllSpider(total=1000000) close_on = 0.1 - yield docrawl(spider, {'CLOSESPIDER_TIMEOUT': close_on}) - reason = spider.meta['close_reason'] + crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on}) + yield crawler.crawl(total=1000000) + reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_timeout') - stats = spider.crawler.stats + stats = crawler.stats start = stats.get_value('start_time') stop = stats.get_value('finish_time') diff = stop - start diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 9401bd0c9..48931d6ff 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -3,7 +3,7 @@ import socket import mock from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.utils.test import docrawl, get_testlog +from scrapy.utils.test import get_crawler, get_testlog from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider from tests.mockserver import MockServer @@ -21,9 +21,9 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_follow_all(self): - spider = FollowAllSpider() - yield docrawl(spider) - self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url + crawler = get_crawler(FollowAllSpider) + yield crawler.crawl() + self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks def test_delay(self): @@ -37,9 +37,9 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def _test_delay(self, delay, randomize): settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} - spider = FollowAllSpider(maxlatency=delay * 2) - yield docrawl(spider, settings) - t = spider.times + crawler = get_crawler(FollowAllSpider, settings) + yield crawler.crawl(maxlatency=delay * 2) + t = crawler.spider.times totaltime = t[-1] - t[0] avgd = totaltime / (len(t) - 1) tolerance = 0.6 if randomize else 0.2 @@ -48,85 +48,79 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_success(self): - spider = DelaySpider(n=0.5) - yield docrawl(spider) - self.assertTrue(spider.t1 > 0) - self.assertTrue(spider.t2 > 0) - self.assertTrue(spider.t2 > spider.t1) + crawler = get_crawler(DelaySpider) + yield crawler.crawl(n=0.5) + self.assertTrue(crawler.spider.t1 > 0) + self.assertTrue(crawler.spider.t2 > 0) + self.assertTrue(crawler.spider.t2 > crawler.spider.t1) @defer.inlineCallbacks def test_timeout_failure(self): - spider = DelaySpider(n=0.5) - yield docrawl(spider, {"DOWNLOAD_TIMEOUT": 0.35}) - self.assertTrue(spider.t1 > 0) - self.assertTrue(spider.t2 == 0) - self.assertTrue(spider.t2_err > 0) - self.assertTrue(spider.t2_err > spider.t1) + crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) + yield crawler.crawl(n=0.5) + self.assertTrue(crawler.spider.t1 > 0) + self.assertTrue(crawler.spider.t2 == 0) + self.assertTrue(crawler.spider.t2_err > 0) + self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) # server hangs after receiving response headers - spider = DelaySpider(n=0.5, b=1) - yield docrawl(spider, {"DOWNLOAD_TIMEOUT": 0.35}) - self.assertTrue(spider.t1 > 0) - self.assertTrue(spider.t2 == 0) - self.assertTrue(spider.t2_err > 0) - self.assertTrue(spider.t2_err > spider.t1) + yield crawler.crawl(n=0.5, b=1) + self.assertTrue(crawler.spider.t1 > 0) + self.assertTrue(crawler.spider.t2 == 0) + self.assertTrue(crawler.spider.t2_err > 0) + self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) @defer.inlineCallbacks def test_retry_503(self): - spider = SimpleSpider("http://localhost:8998/status?n=503") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://localhost:8998/status?n=503") self._assert_retried() @defer.inlineCallbacks def test_retry_conn_failed(self): - spider = SimpleSpider("http://localhost:65432/status?n=503") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://localhost:65432/status?n=503") self._assert_retried() @defer.inlineCallbacks def test_retry_dns_error(self): with mock.patch('socket.gethostbyname', side_effect=socket.gaierror(-5, 'No address associated with hostname')): - spider = SimpleSpider("http://example.com/") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://example.com/") self._assert_retried() @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): - spider = BrokenStartRequestsSpider(fail_before_yield=1) - yield docrawl(spider) + crawler = get_crawler(BrokenStartRequestsSpider) + yield crawler.crawl(fail_before_yield=1) errors = self.flushLoggedErrors(ZeroDivisionError) self.assertEqual(len(errors), 1) @defer.inlineCallbacks def test_start_requests_bug_yielding(self): - spider = BrokenStartRequestsSpider(fail_yielding=1) - yield docrawl(spider) + crawler = get_crawler(BrokenStartRequestsSpider) + yield crawler.crawl(fail_yielding=1) errors = self.flushLoggedErrors(ZeroDivisionError) self.assertEqual(len(errors), 1) @defer.inlineCallbacks def test_start_requests_lazyness(self): settings = {"CONCURRENT_REQUESTS": 1} - spider = BrokenStartRequestsSpider() - yield docrawl(spider, settings) - #self.assertTrue(False, spider.seedsseen) - #self.assertTrue(spider.seedsseen.index(None) < spider.seedsseen.index(99), - # spider.seedsseen) + crawler = get_crawler(BrokenStartRequestsSpider, settings) + yield crawler.crawl() + #self.assertTrue(False, crawler.spider.seedsseen) + #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), + # crawler.spider.seedsseen) @defer.inlineCallbacks def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - spider = DuplicateStartRequestsSpider(dont_filter=True, - distinct_urls=2, - dupe_factor=3) - yield docrawl(spider, settings) - self.assertEqual(spider.visited, 6) + crawler = get_crawler(DuplicateStartRequestsSpider, settings) + yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3) + self.assertEqual(crawler.spider.visited, 6) - spider = DuplicateStartRequestsSpider(dont_filter=False, - distinct_urls=3, - dupe_factor=4) - yield docrawl(spider, settings) - self.assertEqual(spider.visited, 3) + yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4) + self.assertEqual(crawler.spider.visited, 3) @defer.inlineCallbacks def test_unbounded_response(self): @@ -150,23 +144,23 @@ Connection: close foo body with multiples lines '''}) - spider = SimpleSpider("http://localhost:8998/raw?{0}".format(query)) - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://localhost:8998/raw?{0}".format(query)) log = get_testlog() self.assertEqual(log.count("Got response 200"), 1) @defer.inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data - spider = SimpleSpider("http://localhost:8998/drop?abort=0") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://localhost:8998/drop?abort=0") self._assert_retried() @defer.inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data - spider = SimpleSpider("http://localhost:8998/drop?abort=1") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("http://localhost:8998/drop?abort=1") self._assert_retried() def _assert_retried(self): @@ -184,22 +178,22 @@ with multiples lines req0.meta['next'] = req1 req1.meta['next'] = req2 req2.meta['next'] = req3 - spider = SingleRequestSpider(seed=req0) - yield docrawl(spider) + crawler = get_crawler(SingleRequestSpider) + yield crawler.crawl(seed=req0) # basic asserts in case of weird communication errors - self.assertIn('responses', spider.meta) - self.assertNotIn('failures', spider.meta) + self.assertIn('responses', crawler.spider.meta) + self.assertNotIn('failures', crawler.spider.meta) # start requests doesn't set Referer header - echo0 = json.loads(spider.meta['responses'][2].body) + echo0 = json.loads(crawler.spider.meta['responses'][2].body) self.assertNotIn('Referer', echo0['headers']) # following request sets Referer to start request url - echo1 = json.loads(spider.meta['responses'][1].body) + echo1 = json.loads(crawler.spider.meta['responses'][1].body) self.assertEqual(echo1['headers'].get('Referer'), [req0.url]) # next request avoids Referer header - echo2 = json.loads(spider.meta['responses'][2].body) + echo2 = json.loads(crawler.spider.meta['responses'][2].body) self.assertNotIn('Referer', echo2['headers']) # last request explicitly sets a Referer header - echo3 = json.loads(spider.meta['responses'][3].body) + echo3 = json.loads(crawler.spider.meta['responses'][3].body) self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com']) @defer.inlineCallbacks @@ -208,11 +202,11 @@ with multiples lines est = [] def cb(response): - est.append(get_engine_status(spider.crawler.engine)) + est.append(get_engine_status(crawler.engine)) - spider = SingleRequestSpider(seed='http://localhost:8998/', callback_func=cb) - yield docrawl(spider) + crawler = get_crawler(SingleRequestSpider) + yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb) self.assertEqual(len(est), 1, est) s = dict(est[0]) - self.assertEqual(s['engine.spider.name'], spider.name) + self.assertEqual(s['engine.spider.name'], crawler.spider.name) self.assertEqual(s['len(engine.scraper.slot.active)'], 1) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 6a3115004..c444d35fa 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -47,19 +47,22 @@ class LoadTestCase(unittest.TestCase): def test_enabled_handler(self): handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'} - dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._handlers) self.assertNotIn('scheme', dh._notconfigured) def test_not_configured_handler(self): handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'} - dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + dh = DownloadHandlers(crawler) self.assertNotIn('scheme', dh._handlers) self.assertIn('scheme', dh._notconfigured) def test_disabled_handler(self): handlers = {'scheme': None} - dh = DownloadHandlers(get_crawler({'DOWNLOAD_HANDLERS': handlers})) + crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + dh = DownloadHandlers(crawler) self.assertNotIn('scheme', dh._handlers) self.assertNotIn('scheme', dh._notconfigured) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index b7d3594cd..282035f5c 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -12,9 +12,8 @@ class ManagerTestCase(TestCase): settings_dict = None def setUp(self): - self.crawler = get_crawler(self.settings_dict) - self.spider = Spider('foo') - self.spider.set_crawler(self.crawler) + self.crawler = get_crawler(Spider, self.settings_dict) + self.spider = self.crawler._create_spider('foo') self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) # some mw depends on stats collector self.crawler.stats.open_spider(self.spider) diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 3e5ce6052..e73e62538 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -9,8 +9,8 @@ __doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawl'] class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): - self.spider = Spider('foo') - crawler = get_crawler({'AJAXCRAWL_ENABLED': True}) + crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True}) + self.spider = crawler._create_spider('foo') self.mw = AjaxCrawlMiddleware.from_crawler(crawler) def _ajaxcrawlable_body(self): diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index b37a02336..09973b367 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -10,9 +10,8 @@ from scrapy.utils.test import get_crawler class TestDefaultHeadersMiddleware(TestCase): def get_defaults_spider_mw(self): - crawler = get_crawler() - spider = Spider('foo') - spider.set_crawler(crawler) + crawler = get_crawler(Spider) + spider = crawler._create_spider('foo') defaults = dict([(k, [v]) for k, v in \ six.iteritems(crawler.settings.get('DEFAULT_REQUEST_HEADERS'))]) return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler) diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 52a0cc09d..3e3ff2401 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -9,9 +9,8 @@ from scrapy.utils.test import get_crawler class DownloadTimeoutMiddlewareTest(unittest.TestCase): def get_request_spider_mw(self): - crawler = get_crawler() - spider = Spider('foo') - spider.set_crawler(crawler) + crawler = get_crawler(Spider) + spider = crawler._create_spider('foo') request = Request('http://scrapytest.org/') return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 0eb5e7144..1e22ae661 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -24,8 +24,8 @@ class _BaseTest(unittest.TestCase): self.yesterday = email.utils.formatdate(time.time() - 86400) self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) - self.crawler = get_crawler() - self.spider = Spider('example.com') + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('example.com') self.tmpdir = tempfile.mkdtemp() self.request = Request('http://www.example.com', headers={'User-Agent': 'test'}) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 8b871c7bc..beadfd362 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -10,8 +10,8 @@ from scrapy.utils.test import get_crawler class RedirectMiddlewareTest(unittest.TestCase): def setUp(self): - crawler = get_crawler() - self.spider = Spider('foo') + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') self.mw = RedirectMiddleware.from_crawler(crawler) def test_priority_adjust(self): @@ -123,8 +123,8 @@ class RedirectMiddlewareTest(unittest.TestCase): class MetaRefreshMiddlewareTest(unittest.TestCase): def setUp(self): - crawler = get_crawler() - self.spider = Spider('foo') + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') self.mw = MetaRefreshMiddleware.from_crawler(crawler) def _body(self, interval=5, url='http://example.org/newpage'): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 4c771f18e..e3e7c87d6 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -14,8 +14,8 @@ from scrapy.utils.test import get_crawler class RetryTest(unittest.TestCase): def setUp(self): - crawler = get_crawler() - self.spider = Spider('foo') + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') self.mw = RetryMiddleware.from_crawler(crawler) self.mw.max_retry_times = 2 diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index edc26e543..b790ff09a 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -9,8 +9,8 @@ from scrapy.utils.test import get_crawler class TestDownloaderStats(TestCase): def setUp(self): - self.crawler = get_crawler() - self.spider = Spider('scrapytest.org') + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('scrapytest.org') self.mw = DownloaderStats(self.crawler.stats) self.crawler.stats.open_spider(self.spider) diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 5fd5c24be..909d03ba5 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -9,9 +9,8 @@ from scrapy.utils.test import get_crawler class UserAgentMiddlewareTest(TestCase): def get_spider_and_mw(self, default_useragent): - crawler = get_crawler({'USER_AGENT': default_useragent}) - spider = Spider('foo') - spider.set_crawler(crawler) + crawler = get_crawler(Spider, {'USER_AGENT': default_useragent}) + spider = crawler._create_spider('foo') return spider, UserAgentMiddleware.from_crawler(crawler) def test_default_agent(self): diff --git a/tests/test_engine.py b/tests/test_engine.py index 6a0314a02..244d339ef 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -87,20 +87,18 @@ class CrawlerRun(object): self.portno = self.port.getHost().port start_urls = [self.geturl("/"), self.geturl("/redirect")] - self.spider = TestSpider(start_urls=start_urls) for name, signal in vars(signals).items(): if not name.startswith('_'): dispatcher.connect(self.record_signal, signal) - self.crawler = get_crawler() + self.crawler = get_crawler(TestSpider) self.crawler.install() - self.crawler.configure() self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) - self.crawler.crawl(self.spider) - self.crawler.start() + self.crawler.crawl(start_urls=start_urls) + self.spider = self.crawler.spider self.deferred = defer.Deferred() dispatcher.connect(self.stop, signals.engine_stopped) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 8999e102e..5ce48ebf8 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -8,7 +8,7 @@ from netlib import http_auth from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.utils.test import get_testlog, docrawl +from scrapy.utils.test import get_testlog, get_crawler from scrapy.http import Request from tests.spiders import SimpleSpider, SingleRequestSpider from tests.mockserver import MockServer @@ -49,29 +49,29 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_connect_tunnel(self): - spider = SimpleSpider("https://localhost:8999/status?n=200") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("https://localhost:8999/status?n=200") self._assert_got_response_code(200) @defer.inlineCallbacks def test_https_noconnect(self): os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect' - spider = SimpleSpider("https://localhost:8999/status?n=200") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("https://localhost:8999/status?n=200") self._assert_got_response_code(200) os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_connect_tunnel_error(self): - spider = SimpleSpider("https://localhost:99999/status?n=200") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("https://localhost:99999/status?n=200") self._assert_got_tunnel_error() @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888' - spider = SimpleSpider("https://localhost:8999/status?n=200") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("https://localhost:8999/status?n=200") # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. self._assert_got_tunnel_error() @@ -80,17 +80,17 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): request = Request("https://localhost:8999/echo") - spider = SingleRequestSpider(seed=request) - yield docrawl(spider) + crawler = get_crawler(SingleRequestSpider) + yield crawler.crawl(seed=request) self._assert_got_response_code(200) - echo = json.loads(spider.meta['responses'][0].body) + echo = json.loads(crawler.spider.meta['responses'][0].body) self.assertTrue('Proxy-Authorization' not in echo['headers']) @defer.inlineCallbacks def test_https_noconnect_auth_error(self): os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect' - spider = SimpleSpider("https://localhost:8999/status?n=200") - yield docrawl(spider) + crawler = get_crawler(SimpleSpider) + yield crawler.crawl("https://localhost:8999/status?n=200") self._assert_got_response_code(407) def _assert_got_response_code(self, code): diff --git a/tests/test_spider.py b/tests/test_spider.py index 903ea684a..148a872dd 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -242,7 +242,8 @@ class CrawlSpiderTest(SpiderTest): self.assertTrue(hasattr(spider, '_follow_links')) self.assertTrue(spider._follow_links) - crawler.settings.set('CRAWLSPIDER_FOLLOW_LINKS', False) + settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False} + crawler = get_crawler(settings_dict=settings_dict) spider = self.spider_class.from_crawler(crawler, 'example.com') self.assertTrue(hasattr(spider, '_follow_links')) self.assertFalse(spider._follow_links) @@ -256,7 +257,8 @@ class CrawlSpiderTest(SpiderTest): self.assertTrue(spider._follow_links) spider = self.spider_class('example.com') - spider.set_crawler(get_crawler({'CRAWLSPIDER_FOLLOW_LINKS': False})) + settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False} + spider.set_crawler(get_crawler(settings_dict=settings_dict)) self.assertTrue(hasattr(spider, '_follow_links')) self.assertFalse(spider._follow_links) diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 94404ff41..e7ae75ed2 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -10,9 +10,10 @@ from scrapy.utils.test import get_crawler class TestDepthMiddleware(TestCase): def setUp(self): - self.spider = Spider('scrapytest.org') + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('scrapytest.org') - self.stats = StatsCollector(get_crawler()) + self.stats = StatsCollector(crawler) self.stats.open_spider(self.spider) self.mw = DepthMiddleware(1, self.stats, True) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 788a0986b..5394f0eee 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -3,7 +3,7 @@ from unittest import TestCase from twisted.trial.unittest import TestCase as TrialTestCase from twisted.internet import defer -from scrapy.utils.test import docrawl, get_testlog +from scrapy.utils.test import get_crawler, get_testlog from tests.mockserver import MockServer from scrapy.http import Response, Request from scrapy.spider import Spider @@ -165,20 +165,20 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): @defer.inlineCallbacks def test_middleware_works(self): - spider = _HttpErrorSpider() - yield docrawl(spider) - assert not spider.skipped, spider.skipped - self.assertEqual(spider.parsed, {'200'}) - self.assertEqual(spider.failed, {'404', '402', '500'}) + crawler = get_crawler(_HttpErrorSpider) + yield crawler.crawl() + assert not crawler.spider.skipped, crawler.spider.skipped + self.assertEqual(crawler.spider.parsed, {'200'}) + self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @defer.inlineCallbacks def test_logging(self): - spider = _HttpErrorSpider(bypass_status_codes={402}) - yield docrawl(spider) + crawler = get_crawler(_HttpErrorSpider) + yield crawler.crawl(bypass_status_codes={402}) # print(get_testlog()) - self.assertEqual(spider.parsed, {'200', '402'}) - self.assertEqual(spider.skipped, {'402'}) - self.assertEqual(spider.failed, {'404', '500'}) + self.assertEqual(crawler.spider.parsed, {'200', '402'}) + self.assertEqual(crawler.spider.skipped, {'402'}) + self.assertEqual(crawler.spider.failed, {'404', '500'}) log = get_testlog() self.assertIn('Ignoring response <404', log) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 298cba6e4..e5e99002a 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -10,13 +10,13 @@ from scrapy.utils.test import get_crawler class TestOffsiteMiddleware(TestCase): def setUp(self): - self.spider = self._get_spider() - crawler = get_crawler() + crawler = get_crawler(Spider) + self.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) self.mw.spider_opened(self.spider) - def _get_spider(self): - return Spider('foo', allowed_domains=['scrapytest.org', 'scrapy.org']) + def _get_spiderargs(self): + return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org']) def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -39,8 +39,8 @@ class TestOffsiteMiddleware(TestCase): class TestOffsiteMiddleware2(TestOffsiteMiddleware): - def _get_spider(self): - return Spider('foo', allowed_domains=None) + def _get_spiderargs(self): + return dict(name='foo', allowed_domains=None) def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -58,7 +58,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): def _get_spider(self): bad_hostname = urlparse('http:////scrapytest.org').hostname - return Spider('foo', allowed_domains=['scrapytest.org', None, bad_hostname]) + return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname]) def test_process_spider_output(self): res = Response('http://scrapytest.org') diff --git a/tests/test_stats.py b/tests/test_stats.py index 795e8e3bd..db1f50712 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -7,8 +7,8 @@ from scrapy.utils.test import get_crawler class StatsCollectorTest(unittest.TestCase): def setUp(self): - self.crawler = get_crawler() - self.spider = Spider('foo') + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('foo') def test_collector(self): stats = StatsCollector(self.crawler) From d0edad4b0bd93ed34a680ddd6563387be7797128 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 31 Jul 2014 10:10:25 -0300 Subject: [PATCH 10/18] Drop support for ScrapyCommand.crawler property --- scrapy/command.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/scrapy/command.py b/scrapy/command.py index b2eb9cf8f..9ac013098 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -2,12 +2,11 @@ Base class for Scrapy commands """ import os -import warnings from optparse import OptionGroup from twisted.python import failure from scrapy.utils.conf import arglist_to_dict -from scrapy.exceptions import UsageError, ScrapyDeprecationWarning +from scrapy.exceptions import UsageError class ScrapyCommand(object): @@ -27,31 +26,6 @@ class ScrapyCommand(object): assert not hasattr(self, '_crawler'), "crawler already set" self._crawler = crawler - @property - def crawler(self): - warnings.warn("Command's default `crawler` is deprecated and will be removed. " - "Use `create_crawler` method to instatiate crawlers.", - ScrapyDeprecationWarning) - - if not hasattr(self, '_crawler'): - crawler = self.crawler_process.create_crawler() - - old_start = crawler.start - self.crawler_process.started = False - - def wrapped_start(): - if self.crawler_process.started: - old_start() - else: - self.crawler_process.started = True - self.crawler_process.start() - - crawler.start = wrapped_start - - self.set_crawler(crawler) - - return self._crawler - def syntax(self): """ Command syntax (preferably one-line). Do not include command name. From 89df18bd2bc6fdc7f2084454a2c69f4db03008ad Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 31 Jul 2014 10:16:25 -0300 Subject: [PATCH 11/18] Fix usage of crawler_process in ScrapyCommands --- scrapy/commands/bench.py | 4 +--- scrapy/commands/check.py | 32 +++++++++----------------------- scrapy/commands/crawl.py | 4 +--- scrapy/commands/edit.py | 7 +++---- scrapy/commands/genspider.py | 5 ++--- scrapy/commands/list.py | 3 +-- scrapy/commands/runspider.py | 5 ++--- scrapy/contracts/__init__.py | 17 +++++++++++++++++ 8 files changed, 36 insertions(+), 41 deletions(-) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 18934f976..395597546 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -22,9 +22,7 @@ class Command(ScrapyCommand): def run(self, args, opts): with _BenchServer(): - spider = _BenchSpider(total=100000) - crawler = self.crawler_process.create_crawler() - crawler.crawl(spider) + self.crawler_process.crawl(_BenchSpider, total=100000) self.crawler_process.start() diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 40ff9014b..014b00eeb 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -69,20 +69,18 @@ class Command(ScrapyCommand): # contract requests contract_reqs = defaultdict(list) - spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) - spiders = spman_cls.from_settings(self.settings) + spiders = self.crawler_process.spiders - for spider in args or spiders.list(): - spider = spiders.create(spider) - requests = self.get_requests(spider, conman, result) - contract_reqs[spider.name] = [] + for spidername in args or spiders.list(): + spidercls = spiders.load(spidername) + spidercls.start_requests = lambda s: conman.from_spider(s, result) + tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: - for req in requests: - contract_reqs[spider.name].append(req.callback.__name__) - elif requests: - crawler = self.crawler_process.create_crawler(spider.name) - crawler.crawl(spider, requests) + for method in tested_methods: + contract_reqs[spidercls.name].append(method) + elif tested_methods: + self.crawler_process.crawl(spidercls) # start checks if opts.list: @@ -101,15 +99,3 @@ class Command(ScrapyCommand): result.printSummary(start, stop) self.exitcode = int(not result.wasSuccessful()) - def get_requests(self, spider, conman, result): - requests = [] - - for key, value in vars(type(spider)).items(): - if callable(value) and value.__doc__: - bound_method = value.__get__(spider, type(spider)) - request = conman.from_method(bound_method, result) - - if request: - requests.append(request) - - return requests diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 0976de51b..b7fea7b80 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -54,7 +54,5 @@ class Command(ScrapyCommand): raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") spname = args[0] - crawler = self.crawler_process.create_crawler() - spider = crawler.spiders.create(spname, **opts.spargs) - crawler.crawl(spider) + self.crawler_process.crawl(spname, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index b542217e1..e20e7c2e5 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -25,13 +25,12 @@ class Command(ScrapyCommand): if len(args) != 1: raise UsageError() - crawler = self.crawler_process.create_crawler() - editor = crawler.settings['EDITOR'] + editor = self.settings['EDITOR'] try: - spider = crawler.spiders.create(args[0]) + spidercls = self.crawler_process.spiders.load(args[0]) except KeyError: return self._err("Spider not found: %s" % args[0]) - sfile = sys.modules[spider.__module__].__file__ + sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace('.pyc', '.py') self.exitcode = os.system('%s "%s"' % (editor, sfile)) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 3e2e24b21..52c5d9f94 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -65,15 +65,14 @@ class Command(ScrapyCommand): return try: - crawler = self.crawler_process.create_crawler() - spider = crawler.spiders.create(name) + spidercls = self.crawler_process.spiders.load(name) except KeyError: pass else: # if spider already exists and not --force then halt if not opts.force: print("Spider %r already exists in module:" % name) - print(" %s" % spider.__module__) + print(" %s" % spidercls.__module__) return template_file = self._find_template(opts.template) if template_file: diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 0ea9c2313..2d55d59bd 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -10,6 +10,5 @@ class Command(ScrapyCommand): return "List available spiders" def run(self, args, opts): - crawler = self.crawler_process.create_crawler() - for s in sorted(crawler.spiders.list()): + for s in sorted(self.crawler_process.spiders.list()): print(s) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index b1d501842..b67838619 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -83,8 +83,7 @@ class Command(ScrapyCommand): spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError("No spider found in file: %s\n" % filename) - spider = spclasses.pop()(**opts.spargs) + spidercls = spclasses.pop() - crawler = self.crawler_process.create_crawler() - crawler.crawl(spider) + self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 03e6e4e0c..5eaee3d11 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -15,6 +15,15 @@ class ContractsManager(object): for contract in contracts: self.contracts[contract.name] = contract + def tested_methods_from_spidercls(self, spidercls): + methods = [] + for key, value in vars(spidercls).items(): + if (callable(value) and value.__doc__ and + re.search(r'^\s*@', value.__doc__, re.MULTILINE)): + methods.append(key) + + return methods + def extract_contracts(self, method): contracts = [] for line in method.__doc__.split('\n'): @@ -28,6 +37,14 @@ class ContractsManager(object): return contracts + def from_spider(self, spider, results): + requests = [] + for method in self.tested_methods_from_spidercls(type(spider)): + bound_method = spider.__getattribute__(method) + requests.append(self.from_method(bound_method, results)) + + return requests + def from_method(self, method, results): contracts = self.extract_contracts(method) if contracts: From 900a487682b11696ccab5d18c9f13e0addd25f12 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 5 Aug 2014 21:01:57 -0300 Subject: [PATCH 12/18] Support multiple simultaneous LogObservers listening different crawlers --- docs/topics/logging.rst | 13 +++++++++--- scrapy/crawler.py | 5 +++++ scrapy/log.py | 30 +++++++++++++++------------ tests/test_log.py | 46 ++++++++++++++++++++++++++++++++++------- 4 files changed, 71 insertions(+), 23 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 1a9e975d8..819884ac2 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -10,7 +10,11 @@ logging`_ but this may change in the future. .. _Twisted logging: http://twistedmatrix.com/projects/core/documentation/howto/logging.html -The logging service must be explicitly started through the :func:`scrapy.log.start` function. +The logging service must be explicitly started through the +:func:`scrapy.log.start` function to catch the top level Scrapy's log messages. +On top of that, each crawler has its own independent log observer +(automatically attached when it's created) that intercepts its spider's log +messages. .. _topics-logging-levels: @@ -55,8 +59,11 @@ scrapy.log module .. function:: start(logfile=None, loglevel=None, logstdout=None) - Start the logging facility. This must be called before actually logging any - messages. Otherwise, messages logged before this call will get lost. + Start the top level Scrapy logger. This must be called before actually + logging any top level messages (those logged using this module's + :func:`~scrapy.log.msg` function instead of the :meth:`Spider.log + ` method). Otherwise, messages logged before this + call will get lost. :param logfile: the file path to use for logging output. If omitted, the :setting:`LOG_FILE` setting will be used. If both are ``None``, the log diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 56823166b..597bb2e9d 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -128,6 +128,7 @@ class CrawlerProcess(CrawlerRunner): signame = signal_names[signum] log.msg(format='Received %(signame)s twice, forcing unclean shutdown', level=log.INFO, signame=signame) + self._stop_logging() reactor.callFromThread(self._stop_reactor) def start(self, stop_after_crawl=True): @@ -135,6 +136,7 @@ class CrawlerProcess(CrawlerRunner): self._start_reactor(stop_after_crawl) def _start_logging(self): + self.log_observer = log.start_from_settings(self.settings) log.scrapy_info(self.settings) def _start_reactor(self, stop_after_crawl=True): @@ -149,6 +151,9 @@ class CrawlerProcess(CrawlerRunner): reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call + def _stop_logging(self): + self.log_observer.stop() + def _stop_reactor(self, _=None): try: reactor.stop() diff --git a/scrapy/log.py b/scrapy/log.py index 1f32003e7..aa53e3574 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -35,15 +35,16 @@ class ScrapyFileLogObserver(log.FileLogObserver): def __init__(self, f, level=INFO, encoding='utf-8', crawler=None): self.level = level self.encoding = encoding + self.crawler = crawler if crawler: - self.crawler = crawler self.emit = self._emit_with_crawler else: self.emit = self._emit log.FileLogObserver.__init__(self, f) def _emit(self, eventDict): - ev = _adapt_eventdict(eventDict, self.level, self.encoding) + ev = _adapt_eventdict(eventDict, self.level, self.encoding, + self.crawler) if ev is not None: log.FileLogObserver.emit(self, ev) return ev @@ -55,7 +56,8 @@ class ScrapyFileLogObserver(log.FileLogObserver): sname = 'log_count/%s' % level_names.get(level, level) self.crawler.stats.inc_value(sname) -def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', prepend_level=True): +def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', + crawler=None, prepend_level=True): """Adapt Twisted log eventDict making it suitable for logging with a Scrapy log observer. It may return None to indicate that the event should be ignored by a Scrapy log observer. @@ -78,6 +80,12 @@ def _adapt_eventdict(eventDict, log_level=INFO, encoding='utf-8', prepend_level= spider = ev.get('spider') if spider: ev['system'] = unicode_to_str(spider.name, encoding) + if crawler and (not spider or spider.crawler is not crawler): + # ignore events not triggered by own spiders in crawlers' observers + return + if not crawler and spider: + # ignore spiders' events in observers without crawler + return lvlname = level_names.get(level, 'NOLEVEL') message = ev.get('message') @@ -140,18 +148,14 @@ def start_from_settings(settings, crawler=None): settings['LOG_ENCODING'], crawler) def scrapy_info(settings): - log_observer = start_from_settings(settings) - if log_observer: - msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, - settings['BOT_NAME'])) + msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, + settings['BOT_NAME'])) - msg("Optional features available: %s" % ", ".join(scrapy.optional_features), - level=INFO) + msg("Optional features available: %s" % ", ".join(scrapy.optional_features), + level=INFO) - d = dict(overridden_settings(settings)) - msg(format="Overridden settings: %(settings)r", settings=d, level=INFO) - - log_observer.stop() + d = dict(overridden_settings(settings)) + msg(format="Overridden settings: %(settings)r", settings=d, level=INFO) def start_from_crawler(crawler): return start_from_settings(crawler.settings, crawler) diff --git a/tests/test_log.py b/tests/test_log.py index 3263cb42e..113d10004 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -6,6 +6,7 @@ from twisted.trial import unittest from scrapy import log from scrapy.spider import Spider from scrapy.settings import default_settings +from scrapy.utils.test import get_crawler class LogTest(unittest.TestCase): @@ -40,10 +41,10 @@ class ScrapyFileLogObserverTest(unittest.TestCase): log.msg("Hello") self.assertEqual(self.logged(), "[scrapy] INFO: Hello") - def test_msg_spider(self): + def test_msg_ignore_spider(self): spider = Spider("myspider") log.msg("Hello", spider=spider) - self.assertEqual(self.logged(), "[myspider] INFO: Hello") + self.failIf(self.logged()) def test_msg_level1(self): log.msg("Hello", level=log.WARNING) @@ -57,11 +58,6 @@ class ScrapyFileLogObserverTest(unittest.TestCase): log.msg("Hello", level=9999) self.assertEqual(self.logged(), "[scrapy] NOLEVEL: Hello") - def test_msg_level_spider(self): - spider = Spider("myspider") - log.msg("Hello", spider=spider, level=log.WARNING) - self.assertEqual(self.logged(), "[myspider] WARNING: Hello") - def test_msg_encoding(self): log.msg(u"Price: \xa3100") self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xc2\xa3100") @@ -133,5 +129,41 @@ class Latin1ScrapyFileLogObserverTest(ScrapyFileLogObserverTest): # self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xa3") +class CrawlerScrapyFileLogObserverTest(unittest.TestCase): + + def setUp(self): + self.f = BytesIO() + self.crawler = get_crawler(Spider) + self.spider = self.crawler.spider = self.crawler._create_spider('test') + self.log_observer = log.ScrapyFileLogObserver(self.f, log.INFO, + 'utf-8', self.crawler) + self.log_observer.start() + + def tearDown(self): + self.flushLoggedErrors() + self.log_observer.stop() + + def logged(self): + return self.f.getvalue().strip()[25:] + + def test_msg_basic(self): + log.msg("Hello", spider=self.spider) + self.assertEqual(self.logged(), "[test] INFO: Hello") + + def test_msg_ignore_scrapy_channel(self): + log.msg("Hello") + self.failIf(self.logged()) + + def test_msg_ignore_another_crawler(self): + crawler = get_crawler(Spider) + log.msg("Hello", spider=crawler._create_spider('test')) + self.failIf(self.logged()) + + def test_msg_stats_log(self): + assert self.crawler.stats.get_value('log_count/INFO', 0) == 0 + log.msg("Hello", spider=self.spider) + self.assertEqual(self.crawler.stats.get_value('log_count/INFO'), 1) + + if __name__ == "__main__": unittest.main() From 9cbbfd8b04835c40568b687ef8b13d901db988cb Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Wed, 6 Aug 2014 08:51:12 -0300 Subject: [PATCH 13/18] Adjust spiders' utils to new SpiderManager API --- scrapy/commands/fetch.py | 14 +++++------- scrapy/commands/parse.py | 49 +++++++++++++++++++++------------------- scrapy/commands/shell.py | 24 +++++++++++++++----- scrapy/shell.py | 9 +++----- scrapy/utils/spider.py | 18 +++++++++------ 5 files changed, 64 insertions(+), 50 deletions(-) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 373d323c7..ca9fd57f5 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -3,9 +3,8 @@ from w3lib.url import is_url from scrapy.command import ScrapyCommand from scrapy.http import Request -from scrapy.spider import Spider from scrapy.exceptions import UsageError -from scrapy.utils.spider import create_spider_for_request +from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): @@ -48,12 +47,11 @@ class Command(ScrapyCommand): request = Request(args[0], callback=cb, dont_filter=True) request.meta['handle_httpstatus_all'] = True - crawler = self.crawler_process.create_crawler() - spider = None + spidercls = DefaultSpider + spiders = self.crawler_process.spiders if opts.spider: - spider = crawler.spiders.create(opts.spider) + spidercls = spiders.load(opts.spider) else: - spider = create_spider_for_request(crawler.spiders, request, \ - default_spider=Spider('default')) - crawler.crawl(spider, [request]) + spidercls = spidercls_for_request(spiders, request, spidercls) + self.crawler_process.crawl(spidercls, start_requests=lambda: [request]) self.crawler_process.start() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 0867a21a0..01c7fff0a 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,7 +5,7 @@ from scrapy.http import Request from scrapy.item import BaseItem from scrapy.utils import display from scrapy.utils.conf import arglist_to_dict -from scrapy.utils.spider import iterate_spider_output, create_spider_for_request +from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError from scrapy import log @@ -113,41 +113,45 @@ class Command(ScrapyCommand): requests.append(x) return items, requests - def get_callback_from_rules(self, response): - if getattr(self.spider, 'rules', None): - for rule in self.spider.rules: + def get_callback_from_rules(self, spider, response): + if getattr(spider, 'rules', None): + for rule in spider.rules: if rule.link_extractor.matches(response.url) and rule.callback: return rule.callback else: log.msg(format='No CrawlSpider rules found in spider %(spider)r, ' 'please specify a callback to use for parsing', - level=log.ERROR, spider=self.spider.name) + level=log.ERROR, spider=spider.name) - def set_spider(self, url, opts): + def set_spidercls(self, url, opts): + spiders = self.crawler_process.spiders if opts.spider: try: - self.spider = self.pcrawler.spiders.create(opts.spider, **opts.spargs) + self.spidercls = spiders.load(opts.spider) except KeyError: log.msg(format='Unable to find spider: %(spider)s', level=log.ERROR, spider=opts.spider) else: - self.spider = create_spider_for_request(self.pcrawler.spiders, Request(url), **opts.spargs) - if not self.spider: + self.spidercls = spidercls_for_request(spiders, Request(url)) + if not self.spidercls: log.msg(format='Unable to find spider for: %(url)s', level=log.ERROR, url=url) - def start_parsing(self, url, opts): request = Request(url, opts.callback) - request = self.prepare_request(request, opts) + _start_requests = lambda s: [self.prepare_request(s, request, opts)] + self.spidercls.start_requests = _start_requests - self.pcrawler.crawl(self.spider, [request]) + + def start_parsing(self, url, opts): + self.crawler_process.crawl(self.spidercls, **opts.spargs) + self.pcrawler = list(self.crawler_process.crawlers)[0] self.crawler_process.start() if not self.first_response: - log.msg(format='No response downloaded for: %(request)s', - level=log.ERROR, request=request) + log.msg(format='No response downloaded for: %(url)s', + level=log.ERROR, url=url) - def prepare_request(self, request, opts): + def prepare_request(self, spider, request, opts): def callback(response): # memorize first request if not self.first_response: @@ -157,17 +161,17 @@ class Command(ScrapyCommand): cb = response.meta['_callback'] if not cb: if opts.rules and self.first_response == response: - cb = self.get_callback_from_rules(response) + cb = self.get_callback_from_rules(spider, response) else: cb = 'parse' if not callable(cb): - cb_method = getattr(self.spider, cb, None) + cb_method = getattr(spider, cb, None) if callable(cb_method): cb = cb_method else: log.msg(format='Cannot find callback %(callback)r in spider: %(spider)s', - callback=callback, spider=self.spider.name, level=log.ERROR) + callback=callback, spider=spider.name, level=log.ERROR) return # parse items and requests @@ -177,7 +181,7 @@ class Command(ScrapyCommand): if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc for item in items: - itemproc.process_item(item, self.spider) + itemproc.process_item(item, spider) self.add_items(depth, items) self.add_requests(depth, requests) @@ -207,10 +211,9 @@ class Command(ScrapyCommand): else: url = args[0] - # prepare spider - self.pcrawler = self.crawler_process.create_crawler() - self.set_spider(url, opts) + # prepare spidercls + self.set_spidercls(url, opts) - if self.spider and opts.depth > 0: + if self.spidercls and opts.depth > 0: self.start_parsing(url, opts) self.print_results(opts) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index ab170e665..e4d32c314 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -8,6 +8,9 @@ from threading import Thread from scrapy.command import ScrapyCommand from scrapy.shell import Shell +from scrapy.http import Request +from scrapy import log +from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): @@ -38,18 +41,27 @@ class Command(ScrapyCommand): pass def run(self, args, opts): - crawler = self.crawler_process.create_crawler() - url = args[0] if args else None - spider = crawler.spiders.create(opts.spider) if opts.spider else None + spiders = self.crawler_process.spiders - self.crawler_process.start_crawling() + spidercls = DefaultSpider + if opts.spider: + spidercls = spiders.load(opts.spider) + elif url: + spidercls = spidercls_for_request(spiders, Request(url), + spidercls, log_multiple=True) + crawler = self.crawler_process._create_logged_crawler(spidercls) + crawler.engine = crawler._create_engine() + crawler.engine.start() + + self.crawler_process._start_logging() self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url, spider=spider) + shell.start(url=url) def _start_crawler_thread(self): - t = Thread(target=self.crawler_process.start_reactor) + t = Thread(target=self.crawler_process._start_reactor, + kwargs={'stop_after_crawl': False}) t.daemon = True t.start() diff --git a/scrapy/shell.py b/scrapy/shell.py index 74eaef40f..6c48ef186 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -21,7 +21,6 @@ from scrapy.spider import Spider from scrapy.utils.console import start_python_console from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser -from scrapy.utils.spider import create_spider_for_request class Shell(object): @@ -67,11 +66,9 @@ class Shell(object): return self.spider if spider is None: - spider = create_spider_for_request(self.crawler.spiders, - request, - Spider('default'), - log_multiple=True) - spider.set_crawler(self.crawler) + spider = self.crawler.spider or self.crawler._create_spider() + + self.crawler.spider = spider self.crawler.engine.open_spider(spider, close_if_idle=False) self.spider = spider return spider diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 4e43bc13f..b81cf2b9b 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,6 +4,7 @@ import six from scrapy import log from scrapy.item import BaseItem +from scrapy.spider import Spider from scrapy.utils.misc import arg_to_iter @@ -25,21 +26,21 @@ def iter_spider_classes(module): getattr(obj, 'name', None): yield obj -def create_spider_for_request(spidermanager, request, default_spider=None, \ - log_none=False, log_multiple=False, **spider_kwargs): - """Create a spider to handle the given Request. +def spidercls_for_request(spidermanager, request, default_spidercls=None, + log_none=False, log_multiple=False): + """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using - the spider manager) and return a (new) Spider if (and only if) there is + the spider manager) and return a Spider class if (and only if) there is only one Spider able to handle the Request. If multiple spiders (or no spider) are found, it will return the - default_spider passed. It can optionally log if multiple or no spiders + default_spidercls passed. It can optionally log if multiple or no spiders are found. """ snames = spidermanager.find_by_request(request) if len(snames) == 1: - return spidermanager.create(snames[0], **spider_kwargs) + return spidermanager.load(snames[0]) if len(snames) > 1 and log_multiple: log.msg(format='More than one spider can handle: %(request)s - %(snames)s', @@ -49,5 +50,8 @@ def create_spider_for_request(spidermanager, request, default_spider=None, \ log.msg(format='Unable to find spider that handles: %(request)s', level=log.ERROR, request=request) - return default_spider + return default_spidercls + +class DefaultSpider(Spider): + name = 'default' From c90977ca98dd51b93d91739115d843f44e6a8a94 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 8 Aug 2014 06:15:20 -0300 Subject: [PATCH 14/18] Drop support for scrapy.project.crawler (And scrapy.stats consequently) --- conftest.py | 2 +- docs/faq.rst | 31 ------------------------------- docs/topics/shell.rst | 2 +- scrapy/crawler.py | 15 --------------- scrapy/project.py | 18 +++++++++++------- scrapy/shell.py | 5 ++--- scrapy/spider.py | 2 +- scrapy/stats.py | 13 +++++++------ tests/test_engine.py | 2 -- 9 files changed, 23 insertions(+), 67 deletions(-) diff --git a/conftest.py b/conftest.py index 9f9a5bca7..aa27ddd2b 100644 --- a/conftest.py +++ b/conftest.py @@ -4,7 +4,7 @@ from twisted.python import log from scrapy import optional_features -collect_ignore = ["scrapy/stats.py"] +collect_ignore = ["scrapy/stats.py", "scrapy/project.py"] if 'django' not in optional_features: collect_ignore.append("tests/test_djangoitem/models.py") diff --git a/docs/faq.rst b/docs/faq.rst index 47bfede71..1d6c56d97 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -280,37 +280,6 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. - -I'm getting an error: "cannot import name crawler" --------------------------------------------------- - -This is caused by Scrapy changes due to the singletons removal. The error is -most likely raised by a module (extension, middleware, pipeline or spider) in -your Scrapy project that imports ``crawler`` from ``scrapy.project``. For -example:: - - from scrapy.project import crawler - - class SomeExtension(object): - def __init__(self): - self.crawler = crawler - # ... - -This way to access the crawler object is deprecated, the code should be ported -to use ``from_crawler`` class method, for example:: - - class SomeExtension(object): - - @classmethod - def from_crawler(cls, crawler): - o = cls() - o.crawler = crawler - return o - -Scrapy command line tool has some backwards compatibility in place to support -the old import mechanism (with a deprecation warning), but this mechanism may -not work if you use Scrapy differently (for example, as a library). - .. _user agents: http://en.wikipedia.org/wiki/User_agent .. _LIFO: http://en.wikipedia.org/wiki/LIFO .. _DFO order: http://en.wikipedia.org/wiki/Depth-first_search diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 37268c3ca..5c1cfbd47 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -186,7 +186,7 @@ Here's an example of how you would call it from your spider:: # We want to inspect one specific response. if ".org" in response.url: from scrapy.shell import inspect_response - inspect_response(response) + inspect_response(response, self) # Rest of parsing code. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 597bb2e9d..352cff6e5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -31,18 +31,6 @@ class Crawler(object): self.spider = None self.engine = None - def install(self): - # TODO: remove together with scrapy.project.crawler usage - import scrapy.project - assert not hasattr(scrapy.project, 'crawler'), "crawler already installed" - scrapy.project.crawler = self - - def uninstall(self): - # TODO: remove together with scrapy.project.crawler usage - import scrapy.project - assert hasattr(scrapy.project, 'crawler'), "crawler not installed" - del scrapy.project.crawler - @defer.inlineCallbacks def crawl(self, *args, **kwargs): assert not self.crawling, "Crawling already taking place" @@ -84,9 +72,6 @@ class CrawlerRunner(object): crawler = self._create_logged_crawler(spidercls) self.crawlers.add(crawler) - crawler.install() - crawler.signals.connect(crawler.uninstall, signals.engine_stopped) - d = crawler.crawl(*args, **kwargs) self.crawl_deferreds.add(d) return d diff --git a/scrapy/project.py b/scrapy/project.py index bbe947761..d8973a6c7 100644 --- a/scrapy/project.py +++ b/scrapy/project.py @@ -1,13 +1,17 @@ -""" ---------- WARNING: THIS MODULE IS DEPRECATED ----------- -This module is deprecated. If you want to get the Scrapy crawler from your -extension, middleware or pipeline implement the `from_crawler` class method. +""" +Obsolete module, kept for giving a meaningful error message when trying to +import. +""" + +raise ImportError("""scrapy.project usage has become obsolete. + +If you want to get the Scrapy crawler from your extension, middleware or +pipeline implement the `from_crawler` class method (or look up for extending +components that have already done it, such as spiders). For example: @classmethod def from_crawler(cls, crawler): - return cls(crawler) - -""" + return cls(crawler)""") diff --git a/scrapy/shell.py b/scrapy/shell.py index 6c48ef186..8f87fcb41 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -123,10 +123,9 @@ class Shell(object): return isinstance(value, self.relevant_classes) -def inspect_response(response, spider=None): +def inspect_response(response, spider): """Open a shell to inspect the given response""" - from scrapy.project import crawler - Shell(crawler).start(response=response, spider=spider) + Shell(spider.crawler).start(response=response) def _request_deferred(request): diff --git a/scrapy/spider.py b/scrapy/spider.py index df367b700..943925042 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -93,6 +93,6 @@ class ObsoleteClass(object): raise AttributeError(self.message) spiders = ObsoleteClass(""" -"from scrapy.spider import spiders" no longer works - use "from scrapy.project import crawler" and then access crawler.spiders attribute" +"from scrapy.spider import spiders" no longer works - use "from scrapy.spidermanager import SpiderManager" and instantiate it with your project settings" """) diff --git a/scrapy/stats.py b/scrapy/stats.py index b8128dfc2..710601430 100644 --- a/scrapy/stats.py +++ b/scrapy/stats.py @@ -1,7 +1,8 @@ -from scrapy.project import crawler -stats = crawler.stats -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.stats` is deprecated, use `crawler.stats` attribute instead", - ScrapyDeprecationWarning, stacklevel=2) +""" +Obsolete module, kept for giving a meaningful error message when trying to +import. +""" + +raise ImportError("scrapy.stats usage has become obsolete, use " + "`crawler.stats` attribute instead") diff --git a/tests/test_engine.py b/tests/test_engine.py index 244d339ef..67fb8ae79 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -93,7 +93,6 @@ class CrawlerRun(object): dispatcher.connect(self.record_signal, signal) self.crawler = get_crawler(TestSpider) - self.crawler.install() self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) @@ -109,7 +108,6 @@ class CrawlerRun(object): for name, signal in vars(signals).items(): if not name.startswith('_'): disconnect_all(signal) - self.crawler.uninstall() self.deferred.callback(None) def geturl(self, path): From 419026615f040d6277e311b9307a3950f8532468 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 14 Aug 2014 09:19:41 -0300 Subject: [PATCH 15/18] Deprecate Crawler.spiders attribute --- docs/topics/api.rst | 6 ------ scrapy/crawler.py | 18 ++++++++++++++---- tests/py3-ignores.txt | 1 + tests/test_crawler.py | 24 ++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 tests/test_crawler.py diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 0329e2a8f..03a0b4124 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -76,12 +76,6 @@ how you :ref:`configure the downloader middlewares For an introduction on extensions and a list of available extensions on Scrapy see :ref:`topics-extensions`. - .. attribute:: spiders - - The spider manager which takes care of loading spiders. - - Most extensions won't need to access this attribute. - .. attribute:: engine The execution engine, which coordinates the core crawling logic diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 352cff6e5..52e57fe83 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,5 +1,6 @@ import six import signal +import warnings from twisted.internet import reactor, defer @@ -7,6 +8,7 @@ from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.extension import ExtensionManager from scrapy.signalmanager import SignalManager +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object from scrapy import log, signals @@ -23,14 +25,22 @@ class Crawler(object): self.logformatter = lf_cls.from_crawler(self) self.extensions = ExtensionManager.from_crawler(self) - # Attribute kept for backward compatibility (Use CrawlerRunner.spiders) - spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) - self.spiders = spman_cls.from_settings(self.settings) - self.crawling = False self.spider = None self.engine = None + @property + def spiders(self): + if not hasattr(self, '_spiders'): + warnings.warn("Crawler.spiders is deprecated, use " + "CrawlerRunner.spiders or instantiate " + "scrapy.spidermanager.SpiderManager with your " + "settings.", + category=ScrapyDeprecationWarning, stacklevel=2) + spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) + self._spiders = spman_cls.from_settings(self.settings) + return self._spiders + @defer.inlineCallbacks def crawl(self, *args, **kwargs): assert not self.crawling, "Crawling already taking place" diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index ef88eab7e..f3c667cd0 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -8,6 +8,7 @@ tests/test_contrib_exporter.py tests/test_contrib_linkextractors.py tests/test_contrib_loader.py tests/test_crawl.py +tests/test_crawler.py tests/test_djangoitem/__init__.py tests/test_downloader_handlers.py tests/test_downloadermiddleware_ajaxcrawlable.py diff --git a/tests/test_crawler.py b/tests/test_crawler.py new file mode 100644 index 000000000..55381c030 --- /dev/null +++ b/tests/test_crawler.py @@ -0,0 +1,24 @@ +import warnings +import unittest + +from scrapy.crawler import Crawler +from scrapy.settings import Settings +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.misc import load_object + + +class CrawlerTestCase(unittest.TestCase): + + def setUp(self): + self.crawler = Crawler(DefaultSpider, Settings()) + + def test_deprecated_attribute_spiders(self): + with warnings.catch_warnings(record=True) as w: + spiders = self.crawler.spiders + self.assertEqual(len(w), 1) + self.assertIn("Crawler.spiders", str(w[0].message)) + sm_cls = load_object(self.crawler.settings['SPIDER_MANAGER_CLASS']) + self.assertIsInstance(spiders, sm_cls) + + self.crawler.spiders + self.assertEqual(len(w), 1, "Warn deprecated access only once") From 3547ca6e618e19dda86ad1505323b24e82d317bd Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 14 Aug 2014 11:50:33 -0300 Subject: [PATCH 16/18] Add example on running spiders outside projects --- docs/topics/practices.rst | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index e84478d3c..b188ee562 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -32,19 +32,41 @@ project as example. from scrapy.crawler import CrawlerRunner from scrapy.utils.project import get_project_settings - # If you aren't inside a Scrapy project, you could use an instance of the - # Settings class in scrapy.settings instead of the configuration returned - # by get_project_settings runner = CrawlerRunner(get_project_settings()) - # 'followall' is the name of one of the spiders of the project. If you - # aren't working in a Scrapy project, use the spider class as first - # argument instead of its name (or set the SPIDER_MODULES setting so Scrapy - # knows where to look at) + # 'followall' is the name of one of the spiders of the project. d = runner.crawl('followall', domain='scrapinghub.com') d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished +Running spiders outside projects it's not much different. You have to create a +generic :class:`~scrapy.settings.Settings` object and populate it as needed +(See :ref:`topics-settings-ref` for the available settings), instead of using +the configuration returned by `get_project_settings`. + +Spiders can still be referenced by their name if :setting:`SPIDER_MODULES` is +set with the modules where Scrapy should look for spiders. Otherwise, passing +the spider class as first argument in the :meth:`CrawlerRunner.crawl +` method is enough. + +:: + + from twisted.internet import reactor + from scrapy.spider import Spider + from scrapy.crawler import CrawlerRunner + from scrapy.settings import Settings + + class MySpider(Spider): + # Your spider definition + ... + + settings = Settings({'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'}) + runner = CrawlerRunner(settings) + + d = runner.crawl(MySpider) + d.addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until the crawling is finished + .. seealso:: `Twisted Reactor Overview`_. Running multiple spiders in the same process From 70f2010db17b3796c3edce9c6ca3e00c092d6064 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 14 Aug 2014 11:59:25 -0300 Subject: [PATCH 17/18] Change error type when updating frozen settings --- scrapy/settings/__init__.py | 10 +++++++--- tests/test_settings/__init__.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index bbe8ef481..938b93564 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -88,7 +88,7 @@ class Settings(object): return dict(value) def set(self, name, value, priority='project'): - assert not self.frozen, "Trying to modify an immutable Settings object" + self._assert_mutability() if isinstance(priority, six.string_types): priority = SETTINGS_PRIORITIES[priority] if name not in self.attributes: @@ -97,18 +97,22 @@ class Settings(object): self.attributes[name].set(value, priority) def setdict(self, values, priority='project'): - assert not self.frozen, "Trying to modify an immutable Settings object" + self._assert_mutability() for name, value in six.iteritems(values): self.set(name, value, priority) def setmodule(self, module, priority='project'): - assert not self.frozen, "Trying to modify an immutable Settings object" + self._assert_mutability() if isinstance(module, six.string_types): module = import_module(module) for key in dir(module): if key.isupper(): self.set(key, getattr(module, key), priority) + def _assert_mutability(self): + if self.frozen: + raise TypeError("Trying to modify an immutable Settings object") + def copy(self): return copy.deepcopy(self) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index c7e0914d6..38797ad45 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -213,7 +213,7 @@ class SettingsTest(unittest.TestCase): def test_freeze(self): self.settings.freeze() - with self.assertRaises(AssertionError) as cm: + with self.assertRaises(TypeError) as cm: self.settings.set('TEST_BOOL', False) self.assertEqual(str(cm.exception), "Trying to modify an immutable Settings object") From 6339864f95d4ecaed5e428b342277dce9457d954 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 14 Aug 2014 12:32:37 -0300 Subject: [PATCH 18/18] Minor refactor in the docs and functions used in the shell command --- scrapy/commands/shell.py | 6 +++++- scrapy/crawler.py | 8 +++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index e4d32c314..e2ef1545e 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -50,11 +50,15 @@ class Command(ScrapyCommand): elif url: spidercls = spidercls_for_request(spiders, Request(url), spidercls, log_multiple=True) + + # The crawler is created this way since the Shell manually handles the + # crawling engine, so the set up in the crawl method won't work crawler = self.crawler_process._create_logged_crawler(spidercls) + # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() crawler.engine.start() - self.crawler_process._start_logging() + self.crawler_process.start(start_reactor=False) self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 52e57fe83..e0524021a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -126,13 +126,11 @@ class CrawlerProcess(CrawlerRunner): self._stop_logging() reactor.callFromThread(self._stop_reactor) - def start(self, stop_after_crawl=True): - self._start_logging() - self._start_reactor(stop_after_crawl) - - def _start_logging(self): + def start(self, stop_after_crawl=True, start_reactor=True): self.log_observer = log.start_from_settings(self.settings) log.scrapy_info(self.settings) + if start_reactor: + self._start_reactor(stop_after_crawl) def _start_reactor(self, stop_after_crawl=True): if stop_after_crawl: