From a27d91f0a6eb9fd9f2c3ee1d65f131feaf901bd3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sat, 28 Dec 2013 00:47:32 +0600 Subject: [PATCH] Rename BaseSpider to Spider. See GH-495. --- docs/intro/tutorial.rst | 26 +++++++-------- docs/topics/commands.rst | 4 +-- docs/topics/debug.rst | 2 +- docs/topics/downloader-middleware.rst | 6 ++-- docs/topics/item-pipeline.rst | 6 ++-- docs/topics/leaks.rst | 6 ++-- docs/topics/logging.rst | 6 ++-- docs/topics/request-response.rst | 4 +-- docs/topics/selectors.rst | 4 +-- docs/topics/shell.rst | 8 ++--- docs/topics/signals.rst | 16 +++++----- docs/topics/spider-middleware.rst | 12 +++---- docs/topics/spiders.rst | 32 +++++++++---------- extras/qpsclient.py | 4 +-- scrapy/commands/fetch.py | 4 +-- scrapy/contrib/spiders/crawl.py | 4 +-- scrapy/contrib/spiders/feed.py | 6 ++-- scrapy/contrib/spiders/init.py | 6 ++-- scrapy/contrib/spiders/sitemap.py | 4 +-- scrapy/shell.py | 6 ++-- scrapy/spider.py | 12 +++++-- scrapy/templates/spiders/basic.tmpl | 4 +-- scrapy/tests/spiders.py | 4 +-- scrapy/tests/test_commands.py | 10 +++--- scrapy/tests/test_contracts.py | 4 +-- scrapy/tests/test_contrib_feedexport.py | 10 +++--- scrapy/tests/test_contrib_spiderstate.py | 10 +++--- scrapy/tests/test_downloader_handlers.py | 32 +++++++++---------- scrapy/tests/test_downloadermiddleware.py | 4 +-- .../test_downloadermiddleware_cookies.py | 4 +-- ...test_downloadermiddleware_decompression.py | 4 +-- ...est_downloadermiddleware_defaultheaders.py | 4 +-- ...st_downloadermiddleware_downloadtimeout.py | 4 +-- .../test_downloadermiddleware_httpauth.py | 4 +-- .../test_downloadermiddleware_httpcache.py | 4 +-- ...st_downloadermiddleware_httpcompression.py | 4 +-- .../test_downloadermiddleware_httpproxy.py | 4 +-- .../test_downloadermiddleware_redirect.py | 6 ++-- .../tests/test_downloadermiddleware_retry.py | 4 +-- .../tests/test_downloadermiddleware_stats.py | 4 +-- .../test_downloadermiddleware_useragent.py | 4 +-- scrapy/tests/test_engine.py | 4 +-- scrapy/tests/test_log.py | 6 ++-- scrapy/tests/test_logformatter.py | 4 +-- scrapy/tests/test_pipeline_media.py | 4 +-- scrapy/tests/test_settings.py | 2 +- scrapy/tests/test_spider.py | 16 +++++----- .../test_spiders/spider0.py | 4 +-- .../test_spiders/spider1.py | 4 +-- .../test_spiders/spider2.py | 4 +-- .../test_spiders/spider3.py | 4 +-- .../test_spiders/spider4.py | 4 +-- scrapy/tests/test_spidermiddleware_depth.py | 6 ++-- .../tests/test_spidermiddleware_httperror.py | 8 ++--- scrapy/tests/test_spidermiddleware_offsite.py | 8 ++--- scrapy/tests/test_spidermiddleware_referer.py | 4 +-- .../tests/test_spidermiddleware_urllength.py | 6 ++-- scrapy/tests/test_stats.py | 6 ++-- scrapy/tests/test_utils_reqser.py | 4 +-- scrapy/tests/test_utils_serialize.py | 8 ++--- scrapy/tests/test_utils_url.py | 14 ++++---- scrapy/utils/serialize.py | 4 +-- scrapy/utils/spider.py | 4 +-- 63 files changed, 221 insertions(+), 213 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 290aa7e10..246db1a60 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -97,18 +97,18 @@ of domains). They define an initial list of URLs to download, how to follow links, and how to parse the contents of those pages to extract :ref:`items `. -To create a Spider, you must subclass :class:`scrapy.spider.BaseSpider`, and +To create a Spider, you must subclass :class:`scrapy.spider.Spider`, and define the three main, mandatory, attributes: -* :attr:`~scrapy.spider.BaseSpider.name`: identifies the Spider. It must be +* :attr:`~scrapy.spider.Spider.name`: identifies the Spider. It must be unique, that is, you can't set the same name for different Spiders. -* :attr:`~scrapy.spider.BaseSpider.start_urls`: is a list of URLs where the +* :attr:`~scrapy.spider.Spider.start_urls`: is a list of URLs where the Spider will begin to crawl from. So, the first pages downloaded will be those listed here. The subsequent URLs will be generated successively from data contained in the start URLs. -* :meth:`~scrapy.spider.BaseSpider.parse` is a method of the spider, which will +* :meth:`~scrapy.spider.Spider.parse` is a method of the spider, which will be called with the downloaded :class:`~scrapy.http.Response` object of each start URL. The response is passed to the method as the first and only argument. @@ -116,16 +116,16 @@ define the three main, mandatory, attributes: This method is responsible for parsing the response data and extracting scraped data (as scraped items) and more URLs to follow. - The :meth:`~scrapy.spider.BaseSpider.parse` method is in charge of processing + The :meth:`~scrapy.spider.Spider.parse` method is in charge of processing the response and returning scraped data (as :class:`~scrapy.item.Item` objects) and more URLs to follow (as :class:`~scrapy.http.Request` objects). This is the code for our first Spider; save it in a file named ``dmoz_spider.py`` under the ``tutorial/spiders`` directory:: - from scrapy.spider import BaseSpider + from scrapy.spider import Spider - class DmozSpider(BaseSpider): + class DmozSpider(Spider): name = "dmoz" allowed_domains = ["dmoz.org"] start_urls = [ @@ -174,7 +174,7 @@ the spider as their callback function. These Requests are scheduled, then executed, and :class:`scrapy.http.Response` objects are returned and then fed back to the -spider, through the :meth:`~scrapy.spider.BaseSpider.parse` method. +spider, through the :meth:`~scrapy.spider.Spider.parse` method. Extracting Items ---------------- @@ -259,7 +259,7 @@ This is what the shell looks like:: [s] item Item() [s] request [s] response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - [s] spider + [s] spider [s] Useful shortcuts: [s] shelp() Print this help [s] fetch(req_or_url) Fetch a new request or URL and update shell objects @@ -343,10 +343,10 @@ that property here, so:: Let's add this code to our spider:: - from scrapy.spider import BaseSpider + from scrapy.spider import Spider from scrapy.selector import Selector - class DmozSpider(BaseSpider): + class DmozSpider(Spider): name = "dmoz" allowed_domains = ["dmoz.org"] start_urls = [ @@ -386,12 +386,12 @@ Spiders are expected to return their scraped data inside :class:`~scrapy.item.Item` objects. So, in order to return the data we've scraped so far, the final code for our Spider would be like this:: - from scrapy.spider import BaseSpider + from scrapy.spider import Spider from scrapy.selector import Selector from tutorial.items import DmozItem - class DmozSpider(BaseSpider): + class DmozSpider(Spider): name = "dmoz" allowed_domains = ["dmoz.org"] start_urls = [ diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index fe027a0ff..933905a4d 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -190,9 +190,9 @@ Usage example:: xmlfeed $ scrapy genspider -d basic - from scrapy.spider import BaseSpider + from scrapy.spider import Spider - class $classname(BaseSpider): + class $classname(Spider): name = "$name" allowed_domains = ["$domain"] start_urls = ( diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index d651a5013..47a901a6d 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -7,7 +7,7 @@ Debugging Spiders This document explains the most common techniques for debugging spiders. Consider the following scrapy spider below:: - class MySpider(BaseSpider): + class MySpider(Spider): name = 'myspider' start_urls = ( 'http://example.com/page1', diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0a336582c..5f8b1c012 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -91,7 +91,7 @@ single Python class that defines one or more of the following methods: :type request: :class:`~scrapy.http.Request` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object .. method:: process_response(request, response, spider) @@ -118,7 +118,7 @@ single Python class that defines one or more of the following methods: :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object .. method:: process_exception(request, exception, spider) @@ -149,7 +149,7 @@ single Python class that defines one or more of the following methods: :type exception: an ``Exception`` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object .. _topics-downloader-middleware-ref: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index f09153308..7922ee26c 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -37,7 +37,7 @@ single Python class that must implement the following method: :type item: :class:`~scrapy.item.Item` object :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object Additionally, they may also implement the following methods: @@ -46,14 +46,14 @@ Additionally, they may also implement the following methods: This method is called when the spider is opened. :param spider: the spider which was opened - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object .. method:: close_spider(spider) This method is called when the spider is closed. :param spider: the spider which was closed - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object Item pipeline example diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index c60acb47d..27c50a225 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -88,7 +88,7 @@ subclasses): * ``scrapy.http.Response`` * ``scrapy.item.Item`` * ``scrapy.selector.Selector`` -* ``scrapy.spider.BaseSpider`` +* ``scrapy.spider.Spider`` A real example -------------- @@ -150,8 +150,8 @@ difficult to read. For this reason, that function has a ``ignore`` argument which can be used to ignore a particular class (and all its subclases). For example, using:: - >>> from scrapy.spider import BaseSpider - >>> prefs(ignore=BaseSpider) + >>> from scrapy.spider import Spider + >>> prefs(ignore=Spider) Won't show any live references to spiders. diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index e9306e96f..1a9e975d8 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -43,7 +43,7 @@ Logging from Spiders ==================== The recommended way to log from spiders is by using the Spider -:meth:`~scrapy.spider.BaseSpider.log` method, which already populates the +:meth:`~scrapy.spider.Spider.log` method, which already populates the ``spider`` argument of the :func:`scrapy.log.msg` function. The other arguments are passed directly to the :func:`~scrapy.log.msg` function. @@ -83,10 +83,10 @@ scrapy.log module :param level: the log level for this message. See :ref:`topics-logging-levels`. - :param spider: the spider to use for logging this message. This parameter + :param spider: the spider to use for logging this message. This parameter should always be used when logging things related to a particular spider. - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object .. data:: CRITICAL diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 26fa3d977..280e9fd60 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -37,7 +37,7 @@ Request objects request (once its downloaded) as its first parameter. For more information see :ref:`topics-request-response-ref-request-callback-arguments` below. If a Request doesn't specify a callback, the spider's - :meth:`~scrapy.spider.BaseSpider.parse` method will be used. + :meth:`~scrapy.spider.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. :type callback: callable @@ -342,7 +342,7 @@ automatically pre-populated and only override a couple of them, such as the user name and password. You can use the :meth:`FormRequest.from_response` method for this job. Here's an example spider which uses it:: - class LoginSpider(BaseSpider): + class LoginSpider(Spider): name = 'example.com' start_urls = ['http://www.example.com/users/login.php'] diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 38276ab72..3872736ae 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -56,10 +56,10 @@ Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class constructed by passing a `Response` object as first argument, the response's body is what they're going to be "selecting":: - from scrapy.spider import BaseSpider + from scrapy.spider import Spider from scrapy.selector import Selector - class MySpider(BaseSpider): + class MySpider(Spider): # ... def parse(self, response): sel = Selector(response) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index be3411ccb..fd843c0b3 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -72,7 +72,7 @@ content). Those objects are: * ``spider`` - the Spider which is known to handle the URL, or a - :class:`~scrapy.spider.BaseSpider` object if there is no spider found for + :class:`~scrapy.spider.Spider` object if there is no spider found for the current URL * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched @@ -116,7 +116,7 @@ all start with the ``[s]`` prefix):: [s] request [s] response [s] settings - [s] spider + [s] spider [s] Useful shortcuts: [s] shelp() Prints this help. [s] fetch(req_or_url) Fetch a new request or URL and update objects @@ -136,7 +136,7 @@ After that, we can star playing with the objects:: [s] request [s] response <200 http://slashdot.org> [s] settings - [s] spider + [s] spider [s] Useful shortcuts: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects @@ -165,7 +165,7 @@ This can be achieved by using the ``scrapy.shell.inspect_response`` function. Here's an example of how you would call it from your spider:: - class MySpider(BaseSpider): + class MySpider(Spider): ... def parse(self, response): diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 679d01a69..7923db332 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -77,7 +77,7 @@ item_scraped :type response: :class:`~scrapy.http.Response` object :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object item_dropped ------------ @@ -94,7 +94,7 @@ item_dropped :type item: :class:`~scrapy.item.Item` object :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object :param exception: the exception (which must be a :exc:`~scrapy.exceptions.DropItem` subclass) which caused the item @@ -113,7 +113,7 @@ spider_closed This signal supports returning deferreds from their handlers. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object :param reason: a string which describes the reason why the spider was closed. If it was closed because the spider has completed scraping, the reason @@ -137,7 +137,7 @@ spider_opened This signal supports returning deferreds from their handlers. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object spider_idle ----------- @@ -161,7 +161,7 @@ spider_idle This signal does not support returning deferreds from their handlers. :param spider: the spider which has gone idle - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object spider_error ------------ @@ -178,7 +178,7 @@ spider_error :type response: :class:`~scrapy.http.Response` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object response_received @@ -199,7 +199,7 @@ response_received :type request: :class:`~scrapy.http.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object response_downloaded ------------------- @@ -218,6 +218,6 @@ response_downloaded :type request: :class:`~scrapy.http.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object .. _Failure: http://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 332825337..05ad930a9 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -81,7 +81,7 @@ single Python class that defines one or more of the following methods: :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spider.BaseSpider` object + :type spider: :class:`~scrapy.spider.Spider` object .. method:: process_spider_output(response, result, spider) @@ -101,7 +101,7 @@ single Python class that defines one or more of the following methods: :class:`~scrapy.item.Item` objects :param spider: the spider whose result is being processed - :type spider: :class:`~scrapy.item.BaseSpider` object + :type spider: :class:`~scrapy.item.Spider` object .. method:: process_spider_exception(response, exception, spider) @@ -129,7 +129,7 @@ single Python class that defines one or more of the following methods: :type exception: `Exception`_ object :param spider: the spider which raised the exception - :type spider: :class:`scrapy.spider.BaseSpider` object + :type spider: :class:`scrapy.spider.Spider` object .. method:: process_start_requests(start_requests, spider) @@ -156,7 +156,7 @@ single Python class that defines one or more of the following methods: :type start_requests: an iterable of :class:`~scrapy.http.Request` :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.item.BaseSpider` object + :type spider: :class:`~scrapy.item.Spider` object .. _Exception: http://docs.python.org/library/exceptions.html#exceptions.Exception @@ -268,7 +268,7 @@ OffsiteMiddleware Filters out Requests for URLs outside the domains covered by the spider. This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.spider.BaseSpider.allowed_domains` attribute. + spider's :attr:`~scrapy.spider.Spider.allowed_domains` attribute. When your spider returns a request for a domain not belonging to those covered by the spider, this middleware will log a debug message similar to @@ -283,7 +283,7 @@ OffsiteMiddleware will be printed (but only for the first request filtered). If the spider doesn't define an - :attr:`~scrapy.spider.BaseSpider.allowed_domains` attribute, or the + :attr:`~scrapy.spider.Spider.allowed_domains` attribute, or the attribute is empty, the offsite middleware will allow all requests. If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index dfbdcea18..f9cf28efb 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -17,10 +17,10 @@ For spiders, the scraping cycle goes through something like this: those requests. The first requests to perform are obtained by calling the - :meth:`~scrapy.spider.BaseSpider.start_requests` method which (by default) + :meth:`~scrapy.spider.Spider.start_requests` method which (by default) generates :class:`~scrapy.http.Request` for the URLs specified in the - :attr:`~scrapy.spider.BaseSpider.start_urls` and the - :attr:`~scrapy.spider.BaseSpider.parse` method as callback function for the + :attr:`~scrapy.spider.Spider.start_urls` and the + :attr:`~scrapy.spider.Spider.parse` method as callback function for the Requests. 2. In the callback function, you parse the response (web page) and return either @@ -58,7 +58,7 @@ Spider arguments are passed through the :command:`crawl` command using the Spiders receive arguments in their constructors:: - class MySpider(BaseSpider): + class MySpider(Spider): name = 'myspider' def __init__(self, category=None, *args, **kwargs): @@ -93,10 +93,10 @@ with a ``TestItem`` declared in a ``myproject.items`` module:: .. module:: scrapy.spider :synopsis: Spiders base class, spider manager and spider middleware -BaseSpider ----------- +Spider +------ -.. class:: BaseSpider() +.. class:: Spider() This is the simplest spider, and the one from which every other spider must inherit from (either the ones that come bundled with Scrapy, or the ones @@ -178,7 +178,7 @@ BaseSpider The ``parse`` method is in charge of processing the response and returning scraped data and/or more URLs to follow. Other Requests callbacks have - the same requirements as the :class:`BaseSpider` class. + the same requirements as the :class:`Spider` class. This method, as well as any other Request callback, must return an iterable of :class:`~scrapy.http.Request` and/or @@ -194,15 +194,15 @@ BaseSpider spider. For more information see :ref:`topics-logging`. -BaseSpider example -~~~~~~~~~~~~~~~~~~ +Spider example +~~~~~~~~~~~~~~ Let's see an example:: from scrapy import log # This module is useful for printing out debug information - from scrapy.spider import BaseSpider + from scrapy.spider import Spider - class MySpider(BaseSpider): + class MySpider(Spider): name = 'example.com' allowed_domains = ['example.com'] start_urls = [ @@ -217,11 +217,11 @@ Let's see an example:: Another example returning multiples Requests and Items from a single callback:: from scrapy.selector import Selector - from scrapy.spider import BaseSpider + from scrapy.spider import Spider from scrapy.http import Request from myproject.items import MyItem - class MySpider(BaseSpider): + class MySpider(Spider): name = 'example.com' allowed_domains = ['example.com'] start_urls = [ @@ -252,8 +252,8 @@ CrawlSpider it's generic enough for several cases, so you can start from it and override it as needed for more custom functionality, or just implement your own spider. - Apart from the attributes inherited from BaseSpider (that you must - specify), this class supports a new attribute: + Apart from the attributes inherited from Spider (that you must + specify), this class supports a new attribute: .. attribute:: rules diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 4cdaecb2a..7a1baccca 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -7,11 +7,11 @@ usage: """ -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request -class QPSSpider(BaseSpider): +class QPSSpider(Spider): name = 'qps' benchurl = 'http://localhost:8880/' diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index cf3b1ae7e..373d323c7 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -3,7 +3,7 @@ from w3lib.url import is_url from scrapy.command import ScrapyCommand from scrapy.http import Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.exceptions import UsageError from scrapy.utils.spider import create_spider_for_request @@ -54,6 +54,6 @@ class Command(ScrapyCommand): spider = crawler.spiders.create(opts.spider) else: spider = create_spider_for_request(crawler.spiders, request, \ - default_spider=BaseSpider('default')) + default_spider=Spider('default')) crawler.crawl(spider, [request]) self.crawler_process.start() diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index 3e98b9a61..61b97e5a3 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -9,7 +9,7 @@ import copy from scrapy.http import Request, HtmlResponse from scrapy.utils.spider import iterate_spider_output -from scrapy.spider import BaseSpider +from scrapy.spider import Spider def identity(x): return x @@ -27,7 +27,7 @@ class Rule(object): else: self.follow = follow -class CrawlSpider(BaseSpider): +class CrawlSpider(Spider): rules = () diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py index 271b3a97a..fa538f473 100644 --- a/scrapy/contrib/spiders/feed.py +++ b/scrapy/contrib/spiders/feed.py @@ -4,7 +4,7 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.item import BaseItem from scrapy.http import Request from scrapy.utils.iterators import xmliter, csviter @@ -13,7 +13,7 @@ from scrapy.selector import Selector from scrapy.exceptions import NotConfigured, NotSupported -class XMLFeedSpider(BaseSpider): +class XMLFeedSpider(Spider): """ This class intends to be the base class for spiders that scrape from XML feeds. @@ -92,7 +92,7 @@ class XMLFeedSpider(BaseSpider): for (prefix, uri) in self.namespaces: selector.register_namespace(prefix, uri) -class CSVFeedSpider(BaseSpider): +class CSVFeedSpider(Spider): """Spider for parsing CSV feeds. It receives a CSV file in a response; iterates through each of its rows, and calls parse_row with a dict containing each field's data. diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py index 8de12bb23..9c94a7b33 100644 --- a/scrapy/contrib/spiders/init.py +++ b/scrapy/contrib/spiders/init.py @@ -1,9 +1,9 @@ -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.utils.spider import iterate_spider_output -class InitSpider(BaseSpider): +class InitSpider(Spider): """Base Spider with initialization facilities""" - + def start_requests(self): self._postinit_reqs = super(InitSpider, self).start_requests() return iterate_spider_output(self.init_request()) diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py index d8567f7f0..84ae04d08 100644 --- a/scrapy/contrib/spiders/sitemap.py +++ b/scrapy/contrib/spiders/sitemap.py @@ -1,12 +1,12 @@ import re -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request, XmlResponse from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots from scrapy.utils.gz import gunzip, is_gzipped from scrapy import log -class SitemapSpider(BaseSpider): +class SitemapSpider(Spider): sitemap_urls = () sitemap_rules = [('', 'parse')] diff --git a/scrapy/shell.py b/scrapy/shell.py index 401d80c24..500c241f8 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -11,7 +11,7 @@ from twisted.python import threadable from w3lib.url import any_to_uri from scrapy.item import BaseItem -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.selector import Selector from scrapy.utils.spider import create_spider_for_request from scrapy.utils.misc import load_object @@ -24,7 +24,7 @@ from scrapy.exceptions import IgnoreRequest class Shell(object): - relevant_classes = (BaseSpider, Request, Response, BaseItem, + relevant_classes = (Spider, Request, Response, BaseItem, Selector, Settings) def __init__(self, crawler, update_vars=None, code=None): @@ -67,7 +67,7 @@ class Shell(object): if spider is None: spider = create_spider_for_request(self.crawler.spiders, request, - BaseSpider('default'), + Spider('default'), log_multiple=True) spider.set_crawler(self.crawler) self.crawler.engine.open_spider(spider, close_if_idle=False) diff --git a/scrapy/spider.py b/scrapy/spider.py index 25a59036d..c377b4c3a 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -3,14 +3,14 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ - from scrapy import log from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider +from scrapy.utils.deprecate import warn_when_subclassed -class BaseSpider(object_ref): +class Spider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this class. """ @@ -65,6 +65,14 @@ class BaseSpider(object_ref): __repr__ = __str__ +class BaseSpider(Spider): + __metaclass__ = warn_when_subclassed( + 4, # == len([object, object_ref, Spider, BaseSpider]) + "scrapy.spider.BaseSpider was deprecated. " + "Please inherit from scrapy.spider.Spider." + ) + + class ObsoleteClass(object): def __init__(self, message): self.message = message diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 63ddce401..0a1fbd7fd 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -1,6 +1,6 @@ -from scrapy.spider import BaseSpider +from scrapy.spider import Spider -class $classname(BaseSpider): +class $classname(Spider): name = "$name" allowed_domains = ["$domain"] start_urls = ( diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py index 346615309..03649ddb9 100644 --- a/scrapy/tests/spiders.py +++ b/scrapy/tests/spiders.py @@ -5,13 +5,13 @@ Some spiders used for testing and benchmarking import time from urllib import urlencode -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request from scrapy.item import Item from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor -class MetaSpider(BaseSpider): +class MetaSpider(Spider): name = 'meta' diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index a6562de17..f585ba9fc 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -130,9 +130,9 @@ class RunSpiderCommandTest(CommandTest): with open(fname, 'w') as f: f.write(""" from scrapy import log -from scrapy.spider import BaseSpider +from scrapy.spider import Spider -class MySpider(BaseSpider): +class MySpider(Spider): name = 'myspider' def start_requests(self): @@ -153,7 +153,7 @@ class MySpider(BaseSpider): with open(fname, 'w') as f: f.write(""" from scrapy import log -from scrapy.spider import BaseSpider +from scrapy.spider import Spider """) p = self.proc('runspider', fname) log = p.stderr.read() @@ -184,10 +184,10 @@ class ParseCommandTest(CommandTest): with open(fname, 'w') as f: f.write(""" from scrapy import log -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.item import Item -class MySpider(BaseSpider): +class MySpider(Spider): name = '{0}' def parse(self, response): diff --git a/scrapy/tests/test_contracts.py b/scrapy/tests/test_contracts.py index a1a2ae454..595f2e5f8 100644 --- a/scrapy/tests/test_contracts.py +++ b/scrapy/tests/test_contracts.py @@ -2,7 +2,7 @@ from unittest import TextTestRunner from twisted.trial import unittest -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request from scrapy.item import Item, Field from scrapy.contracts import ContractsManager @@ -22,7 +22,7 @@ class ResponseMock(object): url = 'http://scrapy.org' -class TestSpider(BaseSpider): +class TestSpider(Spider): name = 'demo_spider' def returns_request(self, response): diff --git a/scrapy/tests/test_contrib_feedexport.py b/scrapy/tests/test_contrib_feedexport.py index 5c50eb601..cf85ea1e7 100644 --- a/scrapy/tests/test_contrib_feedexport.py +++ b/scrapy/tests/test_contrib_feedexport.py @@ -6,7 +6,7 @@ from twisted.trial import unittest from twisted.internet import defer from w3lib.url import path_to_file_uri -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.contrib.feedexport import IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage from scrapy.utils.test import assert_aws_environ @@ -38,7 +38,7 @@ class FileFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def _assert_stores(self, storage, path): - spider = BaseSpider("default") + spider = Spider("default") file = storage.open(spider) file.write("content") yield storage.store(file) @@ -59,7 +59,7 @@ class FTPFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def _assert_stores(self, storage, path): - spider = BaseSpider("default") + spider = Spider("default") file = storage.open(spider) file.write("content") yield storage.store(file) @@ -81,7 +81,7 @@ class S3FeedStorageTest(unittest.TestCase): from boto import connect_s3 storage = S3FeedStorage(uri) verifyObject(IFeedStorage, storage) - file = storage.open(BaseSpider("default")) + file = storage.open(Spider("default")) file.write("content") yield storage.store(file) u = urlparse.urlparse(uri) @@ -94,7 +94,7 @@ class StdoutFeedStorageTest(unittest.TestCase): def test_store(self): out = StringIO() storage = StdoutFeedStorage('stdout:', _stdout=out) - file = storage.open(BaseSpider("default")) + file = storage.open(Spider("default")) file.write("content") yield storage.store(file) self.assertEqual(out.getvalue(), "content") diff --git a/scrapy/tests/test_contrib_spiderstate.py b/scrapy/tests/test_contrib_spiderstate.py index 5a438917c..8e5897db7 100644 --- a/scrapy/tests/test_contrib_spiderstate.py +++ b/scrapy/tests/test_contrib_spiderstate.py @@ -3,7 +3,7 @@ from datetime import datetime from twisted.trial import unittest from scrapy.contrib.spiderstate import SpiderState -from scrapy.spider import BaseSpider +from scrapy.spider import Spider class SpiderStateTest(unittest.TestCase): @@ -11,7 +11,7 @@ class SpiderStateTest(unittest.TestCase): def test_store_load(self): jobdir = self.mktemp() os.mkdir(jobdir) - spider = BaseSpider(name='default') + spider = Spider(name='default') dt = datetime.now() ss = SpiderState(jobdir) @@ -20,7 +20,7 @@ class SpiderStateTest(unittest.TestCase): spider.state['dt'] = dt ss.spider_closed(spider) - spider2 = BaseSpider(name='default') + spider2 = Spider(name='default') ss2 = SpiderState(jobdir) ss2.spider_opened(spider2) self.assertEqual(spider.state, {'one': 1, 'dt': dt}) @@ -28,8 +28,8 @@ class SpiderStateTest(unittest.TestCase): def test_state_attribute(self): # state attribute must be present if jobdir is not set, to provide a - # consistent interface - spider = BaseSpider(name='default') + # consistent interface + spider = Spider(name='default') ss = SpiderState() ss.spider_opened(spider) self.assertEqual(spider.state, {}) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 14a26fe33..537013434 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -22,7 +22,7 @@ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request from scrapy.settings import Settings from scrapy import optional_features @@ -45,11 +45,11 @@ class FileTestCase(unittest.TestCase): request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') - return self.download_request(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, Spider('foo')).addCallback(_test) def test_non_existent(self): request = Request('file://%s' % self.mktemp()) - d = self.download_request(request, BaseSpider('foo')) + d = self.download_request(request, Spider('foo')) return self.assertFailure(d, IOError) @@ -87,35 +87,35 @@ class HttpTestCase(unittest.TestCase): def test_download(self): request = Request(self.getURL('file')) - d = self.download_request(request, BaseSpider('foo')) + d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, "0123456789") return d def test_download_head(self): request = Request(self.getURL('file'), method='HEAD') - d = self.download_request(request, BaseSpider('foo')) + d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, '') return d def test_redirect_status(self): request = Request(self.getURL('redirect')) - d = self.download_request(request, BaseSpider('foo')) + d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.status) d.addCallback(self.assertEquals, 302) return d def test_redirect_status_head(self): request = Request(self.getURL('redirect'), method='HEAD') - d = self.download_request(request, BaseSpider('foo')) + d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.status) d.addCallback(self.assertEquals, 302) return d @defer.inlineCallbacks def test_timeout_download_from_spider(self): - spider = BaseSpider('foo') + spider = Spider('foo') meta = {'download_timeout': 0.2} # client connects but no data is received request = Request(self.getURL('wait'), meta=meta) @@ -132,7 +132,7 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) - return self.download_request(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, Spider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): def _test(response): @@ -140,9 +140,9 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers.get('Host'), 'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) - return self.download_request(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, Spider('foo')).addCallback(_test) - d = self.download_request(request, BaseSpider('foo')) + d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, 'example.com') return d @@ -150,7 +150,7 @@ class HttpTestCase(unittest.TestCase): def test_payload(self): body = '1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) - d = self.download_request(request, BaseSpider('foo')) + d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, body) return d @@ -211,7 +211,7 @@ class HttpProxyTestCase(unittest.TestCase): http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) - return self.download_request(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, Spider('foo')).addCallback(_test) def test_download_with_proxy_https_noconnect(self): def _test(response): @@ -221,7 +221,7 @@ class HttpProxyTestCase(unittest.TestCase): http_proxy = '%s?noconnect' % self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) - return self.download_request(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, Spider('foo')).addCallback(_test) def test_download_without_proxy(self): def _test(response): @@ -230,7 +230,7 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEquals(response.body, '/path/to/resource') request = Request(self.getURL('path/to/resource')) - return self.download_request(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, Spider('foo')).addCallback(_test) class DeprecatedHttpProxyTestCase(unittest.TestCase): @@ -270,7 +270,7 @@ class S3TestCase(unittest.TestCase): self.AWS_SECRET_ACCESS_KEY, \ httpdownloadhandler=HttpDownloadHandlerMock) self.download_request = s3reqh.download_request - self.spider = BaseSpider('foo') + self.spider = Spider('foo') def test_request_signing1(self): # gets an object from the johnsmith bucket. diff --git a/scrapy/tests/test_downloadermiddleware.py b/scrapy/tests/test_downloadermiddleware.py index acae94480..cbe3c378d 100644 --- a/scrapy/tests/test_downloadermiddleware.py +++ b/scrapy/tests/test_downloadermiddleware.py @@ -2,7 +2,7 @@ from twisted.trial.unittest import TestCase from twisted.python.failure import Failure from scrapy.http import Request, Response -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.utils.test import get_crawler @@ -13,7 +13,7 @@ class ManagerTestCase(TestCase): def setUp(self): self.crawler = get_crawler(self.settings_dict) - self.spider = BaseSpider('foo') + self.spider = Spider('foo') self.spider.set_crawler(self.crawler) self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) # some mw depends on stats collector diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index 5f5e7a3d5..20fcc656d 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -1,14 +1,14 @@ from unittest import TestCase from scrapy.http import Response, Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): def setUp(self): - self.spider = BaseSpider('foo') + self.spider = Spider('foo') self.mw = CookiesMiddleware() def tearDown(self): diff --git a/scrapy/tests/test_downloadermiddleware_decompression.py b/scrapy/tests/test_downloadermiddleware_decompression.py index aaa7d0ab0..33208028a 100644 --- a/scrapy/tests/test_downloadermiddleware_decompression.py +++ b/scrapy/tests/test_downloadermiddleware_decompression.py @@ -1,7 +1,7 @@ from unittest import TestCase, main from scrapy.http import Response, XmlResponse from scrapy.contrib_exp.downloadermiddleware.decompression import DecompressionMiddleware -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.tests import get_testdata from scrapy.utils.test import assert_samelines @@ -22,7 +22,7 @@ class DecompressionMiddlewareTest(TestCase): def setUp(self): self.mw = DecompressionMiddleware() - self.spider = BaseSpider('foo') + self.spider = Spider('foo') def test_known_compression_formats(self): for fmt in self.test_formats: diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index 4784d75cc..bfe570e2b 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.utils.test import get_crawler @@ -10,7 +10,7 @@ class TestDefaultHeadersMiddleware(TestCase): def get_defaults_spider_mw(self): crawler = get_crawler() - spider = BaseSpider('foo') + spider = Spider('foo') spider.set_crawler(crawler) defaults = dict([(k, [v]) for k, v in \ crawler.settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) diff --git a/scrapy/tests/test_downloadermiddleware_downloadtimeout.py b/scrapy/tests/test_downloadermiddleware_downloadtimeout.py index a7e920e78..52a0cc09d 100644 --- a/scrapy/tests/test_downloadermiddleware_downloadtimeout.py +++ b/scrapy/tests/test_downloadermiddleware_downloadtimeout.py @@ -1,7 +1,7 @@ import unittest from scrapy.contrib.downloadermiddleware.downloadtimeout import DownloadTimeoutMiddleware -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request from scrapy.utils.test import get_crawler @@ -10,7 +10,7 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): def get_request_spider_mw(self): crawler = get_crawler() - spider = BaseSpider('foo') + spider = Spider('foo') spider.set_crawler(crawler) request = Request('http://scrapytest.org/') return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler) diff --git a/scrapy/tests/test_downloadermiddleware_httpauth.py b/scrapy/tests/test_downloadermiddleware_httpauth.py index f43401cb0..adfcd802d 100644 --- a/scrapy/tests/test_downloadermiddleware_httpauth.py +++ b/scrapy/tests/test_downloadermiddleware_httpauth.py @@ -2,9 +2,9 @@ import unittest from scrapy.http import Request from scrapy.contrib.downloadermiddleware.httpauth import HttpAuthMiddleware -from scrapy.spider import BaseSpider +from scrapy.spider import Spider -class TestSpider(BaseSpider): +class TestSpider(Spider): http_user = 'foo' http_pass = 'bar' diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index b491efaaa..da769afbd 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -7,7 +7,7 @@ import email.utils from contextlib import contextmanager from scrapy.http import Response, HtmlResponse, Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.settings import Settings from scrapy.exceptions import IgnoreRequest from scrapy.utils.test import get_crawler @@ -24,7 +24,7 @@ class _BaseTest(unittest.TestCase): self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) self.crawler = get_crawler() - self.spider = BaseSpider('example.com') + self.spider = Spider('example.com') self.tmpdir = tempfile.mkdtemp() self.request = Request('http://www.example.com', headers={'User-Agent': 'test'}) diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index c1a02bbae..770fa7c33 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -3,7 +3,7 @@ from os.path import join, abspath, dirname from cStringIO import StringIO from gzip import GzipFile -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.contrib.downloadermiddleware.httpcompression import HttpCompressionMiddleware from scrapy.tests import tests_datadir @@ -22,7 +22,7 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = BaseSpider('foo') + self.spider = Spider('foo') self.mw = HttpCompressionMiddleware() def _getresponse(self, coding): diff --git a/scrapy/tests/test_downloadermiddleware_httpproxy.py b/scrapy/tests/test_downloadermiddleware_httpproxy.py index b794a3a36..58825c6cf 100644 --- a/scrapy/tests/test_downloadermiddleware_httpproxy.py +++ b/scrapy/tests/test_downloadermiddleware_httpproxy.py @@ -5,9 +5,9 @@ from twisted.trial.unittest import TestCase, SkipTest from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider -spider = BaseSpider('foo') +spider = Spider('foo') class TestDefaultHeadersMiddleware(TestCase): diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index bb74522ee..8b871c7bc 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -1,7 +1,7 @@ import unittest from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware, MetaRefreshMiddleware -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response, HtmlResponse from scrapy.utils.test import get_crawler @@ -11,7 +11,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def setUp(self): crawler = get_crawler() - self.spider = BaseSpider('foo') + self.spider = Spider('foo') self.mw = RedirectMiddleware.from_crawler(crawler) def test_priority_adjust(self): @@ -124,7 +124,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): def setUp(self): crawler = get_crawler() - self.spider = BaseSpider('foo') + self.spider = Spider('foo') self.mw = MetaRefreshMiddleware.from_crawler(crawler) def _body(self, interval=5, url='http://example.org/newpage'): diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index c5732dc4d..0ef0505c1 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -6,7 +6,7 @@ from twisted.internet.error import TimeoutError as ServerTimeoutError, \ from scrapy import optional_features from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware from scrapy.xlib.tx import ResponseFailed -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request, Response from scrapy.utils.test import get_crawler @@ -14,7 +14,7 @@ from scrapy.utils.test import get_crawler class RetryTest(unittest.TestCase): def setUp(self): crawler = get_crawler() - self.spider = BaseSpider('foo') + self.spider = Spider('foo') self.mw = RetryMiddleware.from_crawler(crawler) self.mw.max_retry_times = 2 diff --git a/scrapy/tests/test_downloadermiddleware_stats.py b/scrapy/tests/test_downloadermiddleware_stats.py index aab8a2b5e..edc26e543 100644 --- a/scrapy/tests/test_downloadermiddleware_stats.py +++ b/scrapy/tests/test_downloadermiddleware_stats.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.contrib.downloadermiddleware.stats import DownloaderStats from scrapy.http import Request, Response -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.utils.test import get_crawler @@ -10,7 +10,7 @@ class TestDownloaderStats(TestCase): def setUp(self): self.crawler = get_crawler() - self.spider = BaseSpider('scrapytest.org') + self.spider = Spider('scrapytest.org') self.mw = DownloaderStats(self.crawler.stats) self.crawler.stats.open_spider(self.spider) diff --git a/scrapy/tests/test_downloadermiddleware_useragent.py b/scrapy/tests/test_downloadermiddleware_useragent.py index ece55f081..5fd5c24be 100644 --- a/scrapy/tests/test_downloadermiddleware_useragent.py +++ b/scrapy/tests/test_downloadermiddleware_useragent.py @@ -1,6 +1,6 @@ from unittest import TestCase -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware from scrapy.utils.test import get_crawler @@ -10,7 +10,7 @@ class UserAgentMiddlewareTest(TestCase): def get_spider_and_mw(self, default_useragent): crawler = get_crawler({'USER_AGENT': default_useragent}) - spider = BaseSpider('foo') + spider = Spider('foo') spider.set_crawler(crawler) return spider, UserAgentMiddleware.from_crawler(crawler) diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index 342edd798..49a700073 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -21,7 +21,7 @@ from scrapy import signals from scrapy.utils.test import get_crawler from scrapy.xlib.pydispatch import dispatcher from scrapy.tests import tests_datadir -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.item import Item, Field from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor from scrapy.http import Request @@ -32,7 +32,7 @@ class TestItem(Item): url = Field() price = Field() -class TestSpider(BaseSpider): +class TestSpider(Spider): name = "scrapytest.org" allowed_domains = ["scrapytest.org", "localhost"] diff --git a/scrapy/tests/test_log.py b/scrapy/tests/test_log.py index f29e1804e..d62375d57 100644 --- a/scrapy/tests/test_log.py +++ b/scrapy/tests/test_log.py @@ -4,7 +4,7 @@ from twisted.python import log as txlog, failure from twisted.trial import unittest from scrapy import log -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.settings import default_settings class LogTest(unittest.TestCase): @@ -41,7 +41,7 @@ class ScrapyFileLogObserverTest(unittest.TestCase): self.assertEqual(self.logged(), "[scrapy] INFO: Hello") def test_msg_spider(self): - spider = BaseSpider("myspider") + spider = Spider("myspider") log.msg("Hello", spider=spider) self.assertEqual(self.logged(), "[myspider] INFO: Hello") @@ -58,7 +58,7 @@ class ScrapyFileLogObserverTest(unittest.TestCase): self.assertEqual(self.logged(), "[scrapy] NOLEVEL: Hello") def test_msg_level_spider(self): - spider = BaseSpider("myspider") + spider = Spider("myspider") log.msg("Hello", spider=spider, level=log.WARNING) self.assertEqual(self.logged(), "[myspider] WARNING: Hello") diff --git a/scrapy/tests/test_logformatter.py b/scrapy/tests/test_logformatter.py index d4097aff5..adec19afa 100644 --- a/scrapy/tests/test_logformatter.py +++ b/scrapy/tests/test_logformatter.py @@ -1,6 +1,6 @@ import unittest -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request, Response from scrapy.item import Item, Field from scrapy.logformatter import LogFormatter @@ -18,7 +18,7 @@ class LoggingContribTest(unittest.TestCase): def setUp(self): self.formatter = LogFormatter() - self.spider = BaseSpider('default') + self.spider = Spider('default') def test_crawled(self): req = Request("http://www.example.com") diff --git a/scrapy/tests/test_pipeline_media.py b/scrapy/tests/test_pipeline_media.py index 443894feb..e4fae3983 100644 --- a/scrapy/tests/test_pipeline_media.py +++ b/scrapy/tests/test_pipeline_media.py @@ -6,7 +6,7 @@ from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python import log as txlog from scrapy.http import Request, Response -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.utils.request import request_fingerprint from scrapy.contrib.pipeline.media import MediaPipeline from scrapy.utils.signal import disconnect_all @@ -24,7 +24,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): pipeline_class = MediaPipeline def setUp(self): - self.spider = BaseSpider('media.com') + self.spider = Spider('media.com') self.pipe = self.pipeline_class(download_func=_mocked_download_func) self.pipe.open_spider(self.spider) self.info = self.pipe.spiderinfo diff --git a/scrapy/tests/test_settings.py b/scrapy/tests/test_settings.py index 68d30ccc6..dad52aed9 100644 --- a/scrapy/tests/test_settings.py +++ b/scrapy/tests/test_settings.py @@ -2,7 +2,7 @@ import unittest from scrapy.settings import Settings from scrapy.utils.test import get_crawler -from scrapy.spider import BaseSpider +from scrapy.spider import Spider class SettingsTest(unittest.TestCase): diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py index 97076089b..c587d8f3a 100644 --- a/scrapy/tests/test_spider.py +++ b/scrapy/tests/test_spider.py @@ -5,15 +5,15 @@ from cStringIO import StringIO from twisted.trial import unittest -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse from scrapy.contrib.spiders.init import InitSpider from scrapy.contrib.spiders import CrawlSpider, XMLFeedSpider, CSVFeedSpider, SitemapSpider -class BaseSpiderTest(unittest.TestCase): +class SpiderTest(unittest.TestCase): - spider_class = BaseSpider + spider_class = Spider def setUp(self): warnings.simplefilter("always") @@ -43,12 +43,12 @@ class BaseSpiderTest(unittest.TestCase): self.assertRaises(ValueError, self.spider_class, somearg='foo') -class InitSpiderTest(BaseSpiderTest): +class InitSpiderTest(SpiderTest): spider_class = InitSpider -class XMLFeedSpiderTest(BaseSpiderTest): +class XMLFeedSpiderTest(SpiderTest): spider_class = XMLFeedSpider @@ -92,17 +92,17 @@ class XMLFeedSpiderTest(BaseSpiderTest): ], iterator) -class CSVFeedSpiderTest(BaseSpiderTest): +class CSVFeedSpiderTest(SpiderTest): spider_class = CSVFeedSpider -class CrawlSpiderTest(BaseSpiderTest): +class CrawlSpiderTest(SpiderTest): spider_class = CrawlSpider -class SitemapSpiderTest(BaseSpiderTest): +class SitemapSpiderTest(SpiderTest): spider_class = SitemapSpider diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider0.py b/scrapy/tests/test_spidermanager/test_spiders/spider0.py index 442d690d1..f1f19a1eb 100644 --- a/scrapy/tests/test_spidermanager/test_spiders/spider0.py +++ b/scrapy/tests/test_spidermanager/test_spiders/spider0.py @@ -1,4 +1,4 @@ -from scrapy.spider import BaseSpider +from scrapy.spider import Spider -class Spider0(BaseSpider): +class Spider0(Spider): allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider1.py b/scrapy/tests/test_spidermanager/test_spiders/spider1.py index f38af6ba6..16a533ca2 100644 --- a/scrapy/tests/test_spidermanager/test_spiders/spider1.py +++ b/scrapy/tests/test_spidermanager/test_spiders/spider1.py @@ -1,5 +1,5 @@ -from scrapy.spider import BaseSpider +from scrapy.spider import Spider -class Spider1(BaseSpider): +class Spider1(Spider): name = "spider1" allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider2.py b/scrapy/tests/test_spidermanager/test_spiders/spider2.py index 7af1441e7..4af6f7c41 100644 --- a/scrapy/tests/test_spidermanager/test_spiders/spider2.py +++ b/scrapy/tests/test_spidermanager/test_spiders/spider2.py @@ -1,5 +1,5 @@ -from scrapy.spider import BaseSpider +from scrapy.spider import Spider -class Spider2(BaseSpider): +class Spider2(Spider): name = "spider2" allowed_domains = ["scrapy2.org", "scrapy3.org"] diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider3.py b/scrapy/tests/test_spidermanager/test_spiders/spider3.py index ac9229c8c..b3e5f3da7 100644 --- a/scrapy/tests/test_spidermanager/test_spiders/spider3.py +++ b/scrapy/tests/test_spidermanager/test_spiders/spider3.py @@ -1,6 +1,6 @@ -from scrapy.spider import BaseSpider +from scrapy.spider import Spider -class Spider3(BaseSpider): +class Spider3(Spider): name = "spider3" allowed_domains = ['spider3.com'] diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider4.py b/scrapy/tests/test_spidermanager/test_spiders/spider4.py index 6f66ad37e..e883e4d93 100644 --- a/scrapy/tests/test_spidermanager/test_spiders/spider4.py +++ b/scrapy/tests/test_spidermanager/test_spiders/spider4.py @@ -1,6 +1,6 @@ -from scrapy.spider import BaseSpider +from scrapy.spider import Spider -class Spider4(BaseSpider): +class Spider4(Spider): name = "spider4" @classmethod diff --git a/scrapy/tests/test_spidermiddleware_depth.py b/scrapy/tests/test_spidermiddleware_depth.py index beace38f9..94404ff41 100644 --- a/scrapy/tests/test_spidermiddleware_depth.py +++ b/scrapy/tests/test_spidermiddleware_depth.py @@ -1,8 +1,8 @@ from unittest import TestCase -from scrapy.contrib.spidermiddleware.depth import DepthMiddleware +from scrapy.contrib.spidermiddleware.depth import DepthMiddleware from scrapy.http import Response, Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.statscol import StatsCollector from scrapy.utils.test import get_crawler @@ -10,7 +10,7 @@ from scrapy.utils.test import get_crawler class TestDepthMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider('scrapytest.org') + self.spider = Spider('scrapytest.org') self.stats = StatsCollector(get_crawler()) self.stats.open_spider(self.spider) diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/scrapy/tests/test_spidermiddleware_httperror.py index 0fb730c92..f25db582a 100644 --- a/scrapy/tests/test_spidermiddleware_httperror.py +++ b/scrapy/tests/test_spidermiddleware_httperror.py @@ -1,7 +1,7 @@ from unittest import TestCase from scrapy.http import Response, Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings @@ -9,7 +9,7 @@ from scrapy.settings import Settings class TestHttpErrorMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider('foo') + self.spider = Spider('foo') self.mw = HttpErrorMiddleware(Settings({})) self.req = Request('http://scrapytest.org') @@ -47,7 +47,7 @@ class TestHttpErrorMiddlewareSettings(TestCase): """Similar test, but with settings""" def setUp(self): - self.spider = BaseSpider('foo') + self.spider = Spider('foo') self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOWED_CODES': (402,)})) self.req = Request('http://scrapytest.org') @@ -89,7 +89,7 @@ class TestHttpErrorMiddlewareSettings(TestCase): class TestHttpErrorMiddlewareHandleAll(TestCase): def setUp(self): - self.spider = BaseSpider('foo') + self.spider = Spider('foo') self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True})) self.req = Request('http://scrapytest.org') diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index f7523f726..b7f09dd02 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -1,7 +1,7 @@ from unittest import TestCase from scrapy.http import Response, Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware @@ -13,7 +13,7 @@ class TestOffsiteMiddleware(TestCase): self.mw.spider_opened(self.spider) def _get_spider(self): - return BaseSpider('foo', allowed_domains=['scrapytest.org', 'scrapy.org']) + return Spider('foo', allowed_domains=['scrapytest.org', 'scrapy.org']) def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -33,7 +33,7 @@ class TestOffsiteMiddleware(TestCase): class TestOffsiteMiddleware2(TestOffsiteMiddleware): def _get_spider(self): - return BaseSpider('foo', allowed_domains=None) + return Spider('foo', allowed_domains=None) def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -44,5 +44,5 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): class TestOffsiteMiddleware3(TestOffsiteMiddleware2): def _get_spider(self): - return BaseSpider('foo') + return Spider('foo') diff --git a/scrapy/tests/test_spidermiddleware_referer.py b/scrapy/tests/test_spidermiddleware_referer.py index 467c301ba..f408719d2 100644 --- a/scrapy/tests/test_spidermiddleware_referer.py +++ b/scrapy/tests/test_spidermiddleware_referer.py @@ -1,14 +1,14 @@ from unittest import TestCase from scrapy.http import Response, Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.contrib.spidermiddleware.referer import RefererMiddleware class TestRefererMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider('foo') + self.spider = Spider('foo') self.mw = RefererMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_spidermiddleware_urllength.py b/scrapy/tests/test_spidermiddleware_urllength.py index e3d8c5474..ca2de18f2 100644 --- a/scrapy/tests/test_spidermiddleware_urllength.py +++ b/scrapy/tests/test_spidermiddleware_urllength.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.contrib.spidermiddleware.urllength import UrlLengthMiddleware from scrapy.http import Response, Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider class TestUrlLengthMiddleware(TestCase): @@ -12,10 +12,10 @@ class TestUrlLengthMiddleware(TestCase): short_url_req = Request('http://scrapytest.org/') long_url_req = Request('http://scrapytest.org/this_is_a_long_url') - reqs = [short_url_req, long_url_req] + reqs = [short_url_req, long_url_req] mw = UrlLengthMiddleware(maxlength=25) - spider = BaseSpider('foo') + spider = Spider('foo') out = list(mw.process_spider_output(res, reqs, spider)) self.assertEquals(out, [short_url_req]) diff --git a/scrapy/tests/test_stats.py b/scrapy/tests/test_stats.py index 1b42b9841..795e8e3bd 100644 --- a/scrapy/tests/test_stats.py +++ b/scrapy/tests/test_stats.py @@ -1,6 +1,6 @@ -import unittest +import unittest -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.statscol import StatsCollector, DummyStatsCollector from scrapy.utils.test import get_crawler @@ -8,7 +8,7 @@ class StatsCollectorTest(unittest.TestCase): def setUp(self): self.crawler = get_crawler() - self.spider = BaseSpider('foo') + self.spider = Spider('foo') def test_collector(self): stats = StatsCollector(self.crawler) diff --git a/scrapy/tests/test_utils_reqser.py b/scrapy/tests/test_utils_reqser.py index 31d512d08..4ddc2f472 100644 --- a/scrapy/tests/test_utils_reqser.py +++ b/scrapy/tests/test_utils_reqser.py @@ -1,7 +1,7 @@ import unittest from scrapy.http import Request -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.utils.reqser import request_to_dict, request_from_dict class RequestSerializationTest(unittest.TestCase): @@ -67,7 +67,7 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_to_dict, r) -class TestSpider(BaseSpider): +class TestSpider(Spider): name = 'test' def parse_item(self, response): pass diff --git a/scrapy/tests/test_utils_serialize.py b/scrapy/tests/test_utils_serialize.py index 5bfb87c77..1335dc122 100644 --- a/scrapy/tests/test_utils_serialize.py +++ b/scrapy/tests/test_utils_serialize.py @@ -6,7 +6,7 @@ from decimal import Decimal from twisted.internet import defer from scrapy.utils.serialize import SpiderReferencer, ScrapyJSONEncoder, ScrapyJSONDecoder -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request, Response @@ -21,8 +21,8 @@ class CrawlerMock(object): class BaseTestCase(unittest.TestCase): def setUp(self): - self.spider1 = BaseSpider('name1') - self.spider2 = BaseSpider('name2') + self.spider1 = Spider('name1') + self.spider2 = Spider('name2') open_spiders = set([self.spider1, self.spider2]) crawler = CrawlerMock(open_spiders) self.spref = SpiderReferencer(crawler) @@ -43,7 +43,7 @@ class SpiderReferencerTestCase(BaseTestCase): sp1 = self.spref.get_spider_from_reference(ref1) sp2 = self.spref.get_spider_from_reference(ref2) sp1_ = self.spref.get_spider_from_reference(ref1) - assert isinstance(sp1, BaseSpider) + assert isinstance(sp1, Spider) assert sp1 is not sp2 assert sp1 is sp1_ diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index 3bc56b5c3..a47bb883d 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -1,6 +1,6 @@ import unittest -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url __doctests__ = ['scrapy.utils.url'] @@ -26,14 +26,14 @@ class UrlUtilsTest(unittest.TestCase): self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com'])) def test_url_is_from_spider(self): - spider = BaseSpider(name='example.com') + spider = Spider(name='example.com') self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider)) self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', spider)) self.assertFalse(url_is_from_spider('http://www.example.net/some/page.html', spider)) def test_url_is_from_spider_class_attributes(self): - class MySpider(BaseSpider): + class MySpider(Spider): name = 'example.com' self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider)) self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider)) @@ -41,7 +41,7 @@ class UrlUtilsTest(unittest.TestCase): self.assertFalse(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) def test_url_is_from_spider_with_allowed_domains(self): - spider = BaseSpider(name='example.com', allowed_domains=['example.org', 'example.net']) + spider = Spider(name='example.com', allowed_domains=['example.org', 'example.net']) self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider)) self.assertTrue(url_is_from_spider('http://example.com/some/page.html', spider)) @@ -49,14 +49,14 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider)) - spider = BaseSpider(name='example.com', allowed_domains=set(('example.com', 'example.net'))) + spider = Spider(name='example.com', allowed_domains=set(('example.com', 'example.net'))) self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) - spider = BaseSpider(name='example.com', allowed_domains=('example.com', 'example.net')) + spider = Spider(name='example.com', allowed_domains=('example.com', 'example.net')) self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) def test_url_is_from_spider_with_allowed_domains_class_attributes(self): - class MySpider(BaseSpider): + class MySpider(Spider): name = 'example.com' allowed_domains = ('example.org', 'example.net') self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider)) diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index c60221e4c..95f985872 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -5,7 +5,7 @@ import json from twisted.internet import defer -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request, Response from scrapy.item import BaseItem @@ -42,7 +42,7 @@ class SpiderReferencer(object): def encode_references(self, obj): """Look for Spider objects and replace them with spider references""" - if isinstance(obj, BaseSpider): + if isinstance(obj, Spider): return self.get_reference_from_spider(obj) elif isinstance(obj, dict): d = {} diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 5a9a419a7..d97b8a841 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -14,11 +14,11 @@ def iter_spider_classes(module): """ # this needs to be imported here until get rid of the spider manager # singleton in scrapy.spider.spiders - from scrapy.spider import BaseSpider + from scrapy.spider import Spider for obj in vars(module).itervalues(): if inspect.isclass(obj) and \ - issubclass(obj, BaseSpider) and \ + issubclass(obj, Spider) and \ obj.__module__ == module.__name__ and \ getattr(obj, 'name', None): yield obj