From d3f576a816b89260ea675aaf9d2a0140edd69949 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Sat, 9 May 2015 04:20:09 -0300 Subject: [PATCH] Move scrapy/spider.py to scrapy/spiders/__init__.py --- docs/intro/tutorial.rst | 14 ++++++------- docs/topics/api.rst | 2 +- docs/topics/downloader-middleware.rst | 6 +++--- docs/topics/item-pipeline.rst | 6 +++--- docs/topics/leaks.rst | 4 ++-- docs/topics/logging.rst | 2 +- docs/topics/request-response.rst | 2 +- docs/topics/settings.rst | 2 +- docs/topics/shell.rst | 2 +- docs/topics/signals.rst | 20 +++++++++---------- docs/topics/spider-middleware.rst | 12 +++++------ docs/topics/spiders.rst | 11 +++++----- extras/qpsclient.py | 2 +- scrapy/__init__.py | 2 +- scrapy/crawler.py | 2 +- scrapy/shell.py | 2 +- scrapy/{spider.py => spiders/__init__.py} | 4 ++++ scrapy/spiders/crawl.py | 2 +- scrapy/spiders/feed.py | 2 +- scrapy/spiders/init.py | 6 +++--- scrapy/spiders/sitemap.py | 2 +- scrapy/utils/spider.py | 4 ++-- scrapy/utils/test.py | 2 +- tests/spiders.py | 2 +- tests/test_commands.py | 2 +- tests/test_contracts.py | 2 +- tests/test_downloader_handlers.py | 2 +- tests/test_downloadermiddleware.py | 2 +- ...test_downloadermiddleware_ajaxcrawlable.py | 2 +- tests/test_downloadermiddleware_cookies.py | 2 +- ...test_downloadermiddleware_decompression.py | 2 +- ...est_downloadermiddleware_defaultheaders.py | 2 +- ...st_downloadermiddleware_downloadtimeout.py | 2 +- tests/test_downloadermiddleware_httpauth.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 2 +- ...st_downloadermiddleware_httpcompression.py | 2 +- tests/test_downloadermiddleware_httpproxy.py | 2 +- tests/test_downloadermiddleware_redirect.py | 2 +- tests/test_downloadermiddleware_retry.py | 2 +- tests/test_downloadermiddleware_stats.py | 2 +- tests/test_downloadermiddleware_useragent.py | 2 +- tests/test_engine.py | 2 +- tests/test_logformatter.py | 2 +- tests/test_pipeline_media.py | 2 +- tests/test_spider.py | 5 ++--- tests/test_spiderloader/__init__.py | 2 +- .../test_spiderloader/test_spiders/spider0.py | 2 +- .../test_spiderloader/test_spiders/spider1.py | 2 +- .../test_spiderloader/test_spiders/spider2.py | 2 +- .../test_spiderloader/test_spiders/spider3.py | 2 +- tests/test_spidermiddleware_depth.py | 2 +- tests/test_spidermiddleware_httperror.py | 2 +- tests/test_spidermiddleware_offsite.py | 2 +- tests/test_spidermiddleware_referer.py | 2 +- tests/test_spidermiddleware_urllength.py | 2 +- tests/test_spiderstate.py | 2 +- tests/test_stats.py | 2 +- tests/test_toplevel.py | 2 +- tests/test_utils_reqser.py | 2 +- tests/test_utils_url.py | 2 +- 60 files changed, 96 insertions(+), 94 deletions(-) rename scrapy/{spider.py => spiders/__init__.py} (94%) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 219616587..0d3c49750 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -95,18 +95,18 @@ domain (or group of domains). They define an initial list of URLs to download, how to follow links, and how to parse the contents of pages to extract :ref:`items `. -To create a Spider, you must subclass :class:`scrapy.Spider ` and -define some attributes: +To create a Spider, you must subclass :class:`scrapy.Spider +` and define some attributes: -* :attr:`~scrapy.spider.Spider.name`: identifies the Spider. It must be +* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be unique, that is, you can't set the same name for different Spiders. -* :attr:`~scrapy.spider.Spider.start_urls`: a list of URLs where the +* :attr:`~scrapy.spiders.Spider.start_urls`: a list of URLs where the Spider will begin to crawl from. The first pages downloaded will be those listed here. The subsequent URLs will be generated successively from data contained in the start URLs. -* :meth:`~scrapy.spider.Spider.parse`: a method of the spider, which will +* :meth:`~scrapy.spiders.Spider.parse`: a method of the spider, which will be called with the downloaded :class:`~scrapy.http.Response` object of each start URL. The response is passed to the method as the first and only argument. @@ -114,7 +114,7 @@ define some attributes: This method is responsible for parsing the response data and extracting scraped data (as scraped items) and more URLs to follow. - The :meth:`~scrapy.spider.Spider.parse` method is in charge of processing + The :meth:`~scrapy.spiders.Spider.parse` method is in charge of processing the response and returning scraped data (as :class:`~scrapy.item.Item` objects) and more URLs to follow (as :class:`~scrapy.http.Request` objects). @@ -178,7 +178,7 @@ them the ``parse`` method of the spider as their callback function. These Requests are scheduled, then executed, and :class:`scrapy.http.Response` objects are returned and then fed back to the spider, through the -:meth:`~scrapy.spider.Spider.parse` method. +:meth:`~scrapy.spiders.Spider.parse` method. Extracting Items ---------------- diff --git a/docs/topics/api.rst b/docs/topics/api.rst index ce28b8bc1..e59fe9a58 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -31,7 +31,7 @@ how you :ref:`configure the downloader middlewares .. class:: Crawler(spidercls, settings) The Crawler object must be instantiated with a - :class:`scrapy.spider.Spider` subclass and a + :class:`scrapy.spiders.Spider` subclass and a :class:`scrapy.settings.Settings` object. .. attribute:: settings diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 5cb6c9824..03c5deffe 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -91,7 +91,7 @@ more of the following methods: :type request: :class:`~scrapy.http.Request` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object .. method:: process_response(request, response, spider) @@ -118,7 +118,7 @@ more of the following methods: :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object .. method:: process_exception(request, exception, spider) @@ -149,7 +149,7 @@ more of the following methods: :type exception: an ``Exception`` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object .. _topics-downloader-middleware-ref: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index dd2d79989..f74400b4d 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -36,7 +36,7 @@ Each item pipeline component is a Python class that must implement the following :type item: :class:`~scrapy.item.Item` object or a dict :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object Additionally, they may also implement the following methods: @@ -45,14 +45,14 @@ Additionally, they may also implement the following methods: This method is called when the spider is opened. :param spider: the spider which was opened - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object .. method:: close_spider(self, spider) This method is called when the spider is closed. :param spider: the spider which was closed - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object .. method:: from_crawler(cls, crawler) diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 9d15ebe48..735137ea2 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -92,7 +92,7 @@ subclasses): * :class:`scrapy.http.Response` * :class:`scrapy.item.Item` * :class:`scrapy.selector.Selector` -* :class:`scrapy.spider.Spider` +* :class:`scrapy.spiders.Spider` A real example -------------- @@ -155,7 +155,7 @@ For this reason, that function has a ``ignore`` argument which can be used to ignore a particular class (and all its subclases). For example, this won't show any live references to spiders:: - >>> from scrapy.spider import Spider + >>> from scrapy.spiders import Spider >>> prefs(ignore=Spider) .. module:: scrapy.utils.trackref diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 2cb719998..1a3f5d69f 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -94,7 +94,7 @@ path:: Logging from Spiders ==================== -Scrapy provides a :data:`~scrapy.spider.Spider.logger` within each Spider +Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider instance, that can be accessed and used like this:: import scrapy diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1d695a5f2..aa601f83a 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -37,7 +37,7 @@ Request objects request (once its downloaded) as its first parameter. For more information see :ref:`topics-request-response-ref-request-callback-arguments` below. If a Request doesn't specify a callback, the spider's - :meth:`~scrapy.spider.Spider.parse` method will be used. + :meth:`~scrapy.spiders.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. :type callback: callable diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 26a6d762d..5917bea4e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -67,7 +67,7 @@ Example:: Spiders (See the :ref:`topics-spiders` chapter for reference) can define their own settings that will take precedence and override the project ones. They can -do so by setting their :attr:`scrapy.spider.Spider.custom_settings` attribute. +do so by setting their :attr:`scrapy.spiders.Spider.custom_settings` attribute. 3. Project settings module -------------------------- diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 3b875fec5..9c9411d6d 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -74,7 +74,7 @@ Those objects are: * ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. * ``spider`` - the Spider which is known to handle the URL, or a - :class:`~scrapy.spider.Spider` object if there is no spider found for + :class:`~scrapy.spiders.Spider` object if there is no spider found for the current URL * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 85cf43c76..7ea9efe49 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -74,7 +74,7 @@ item_scraped :type item: dict or :class:`~scrapy.item.Item` object :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object :param response: the response from where the item was scraped :type response: :class:`~scrapy.http.Response` object @@ -94,7 +94,7 @@ item_dropped :type item: dict or :class:`~scrapy.item.Item` object :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object :param response: the response from where the item was dropped :type response: :class:`~scrapy.http.Response` object @@ -116,7 +116,7 @@ spider_closed This signal supports returning deferreds from their handlers. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object :param reason: a string which describes the reason why the spider was closed. If it was closed because the spider has completed scraping, the reason @@ -140,7 +140,7 @@ spider_opened This signal supports returning deferreds from their handlers. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object spider_idle ----------- @@ -164,7 +164,7 @@ spider_idle This signal does not support returning deferreds from their handlers. :param spider: the spider which has gone idle - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object spider_error ------------ @@ -181,7 +181,7 @@ spider_error :type response: :class:`~scrapy.http.Response` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object request_scheduled ----------------- @@ -198,7 +198,7 @@ request_scheduled :type request: :class:`~scrapy.http.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object request_dropped ----------------- @@ -215,7 +215,7 @@ request_dropped :type request: :class:`~scrapy.http.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object response_received ----------------- @@ -235,7 +235,7 @@ response_received :type request: :class:`~scrapy.http.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object response_downloaded ------------------- @@ -254,6 +254,6 @@ response_downloaded :type request: :class:`~scrapy.http.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object .. _Failure: http://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 6e82333f5..84daaaa55 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -81,7 +81,7 @@ following methods: :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object .. method:: process_spider_output(response, result, spider) @@ -102,7 +102,7 @@ following methods: or :class:`~scrapy.item.Item` objects :param spider: the spider whose result is being processed - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object .. method:: process_spider_exception(response, exception, spider) @@ -130,7 +130,7 @@ following methods: :type exception: `Exception`_ object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object .. method:: process_start_requests(start_requests, spider) @@ -157,7 +157,7 @@ following methods: :type start_requests: an iterable of :class:`~scrapy.http.Request` :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.spider.Spider` object + :type spider: :class:`~scrapy.spiders.Spider` object .. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception @@ -272,7 +272,7 @@ OffsiteMiddleware Filters out Requests for URLs outside the domains covered by the spider. This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.spider.Spider.allowed_domains` attribute. + spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute. When your spider returns a request for a domain not belonging to those covered by the spider, this middleware will log a debug message similar to @@ -287,7 +287,7 @@ OffsiteMiddleware will be printed (but only for the first request filtered). If the spider doesn't define an - :attr:`~scrapy.spider.Spider.allowed_domains` attribute, or the + :attr:`~scrapy.spiders.Spider.allowed_domains` attribute, or the attribute is empty, the offsite middleware will allow all requests. If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index d2fdd61b8..025d527a6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -17,10 +17,10 @@ For spiders, the scraping cycle goes through something like this: those requests. The first requests to perform are obtained by calling the - :meth:`~scrapy.spider.Spider.start_requests` method which (by default) + :meth:`~scrapy.spiders.Spider.start_requests` method which (by default) generates :class:`~scrapy.http.Request` for the URLs specified in the - :attr:`~scrapy.spider.Spider.start_urls` and the - :attr:`~scrapy.spider.Spider.parse` method as callback function for the + :attr:`~scrapy.spiders.Spider.start_urls` and the + :attr:`~scrapy.spiders.Spider.parse` method as callback function for the Requests. 2. In the callback function, you parse the response (web page) and return either @@ -42,7 +42,7 @@ Even though this cycle applies (more or less) to any kind of spider, there are different kinds of default spiders bundled into Scrapy for different purposes. We will talk about those types here. -.. module:: scrapy.spider +.. module:: scrapy.spiders :synopsis: Spiders base class, spider manager and spider middleware .. _topics-spiders-ref: @@ -319,8 +319,7 @@ with a ``TestItem`` declared in a ``myproject.items`` module:: description = scrapy.Field() -.. module:: scrapy.spiders - :synopsis: Collection of generic spiders +.. currentmodule:: scrapy.spiders CrawlSpider ----------- diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 7a1baccca..bb83588dd 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -7,7 +7,7 @@ usage: """ -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 31ad2ff02..10ba9544f 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -45,7 +45,7 @@ if twisted_version >= (11, 1, 0): optional_features.add('http11') # Declare top-level shortcuts -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request, FormRequest from scrapy.selector import Selector from scrapy.item import Item, Field diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 161ca4614..e6f4b225f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -138,7 +138,7 @@ class CrawlerRunner(object): :param crawler_or_spidercls: already created crawler, or a spider class or spider's name inside the project to create it :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, - :class:`~scrapy.spider.Spider` subclass or string + :class:`~scrapy.spiders.Spider` subclass or string :param list args: arguments to initialize the spider diff --git a/scrapy/shell.py b/scrapy/shell.py index 8f87fcb41..4142396ab 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -17,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.item import BaseItem from scrapy.settings import Settings -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.utils.console import start_python_console from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser diff --git a/scrapy/spider.py b/scrapy/spiders/__init__.py similarity index 94% rename from scrapy/spider.py rename to scrapy/spiders/__init__.py index 36623b6e2..c08bb964a 100644 --- a/scrapy/spider.py +++ b/scrapy/spiders/__init__.py @@ -111,3 +111,7 @@ spiders = ObsoleteClass( 'it with your project settings"' ) +# Top-level imports +from scrapy.spiders.crawl import CrawlSpider, Rule +from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider +from scrapy.spiders.sitemap import SitemapSpider diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 7dc3dacd6..77551753e 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -9,7 +9,7 @@ import copy from scrapy.http import Request, HtmlResponse from scrapy.utils.spider import iterate_spider_output -from scrapy.spider import Spider +from scrapy.spiders import Spider def identity(x): return x diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index d83ee605e..06e212e1c 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,7 +4,7 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.utils.iterators import xmliter, csviter from scrapy.utils.spider import iterate_spider_output from scrapy.selector import Selector diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 9c94a7b33..7717c8819 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,4 +1,4 @@ -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output class InitSpider(Spider): @@ -20,8 +20,8 @@ class InitSpider(Spider): is called this spider is considered initialized. If you need to perform several requests for initializing your spider, you can do so by using different callbacks. The only requirement is that the final callback - (of the last initialization request) must be self.initialized. - + (of the last initialization request) must be self.initialized. + The default implementation calls self.initialized immediately, and means that no initialization is needed. This method should be overridden only when you need to perform requests to initialize your diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 90fb9fb4e..5aa0b944d 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,7 +1,7 @@ import re import logging -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots from scrapy.utils.gz import gunzip, is_gzipped diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 7ed2d0c3b..94b24f67e 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -3,7 +3,7 @@ import inspect import six -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.utils.misc import arg_to_iter logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ def iter_spider_classes(module): """ # this needs to be imported here until get rid of the spider manager # singleton in scrapy.spider.spiders - from scrapy.spider import Spider + from scrapy.spiders import Spider for obj in six.itervalues(vars(module)): if inspect.isclass(obj) and \ diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index ad4a6aa7c..bec9bdda9 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -27,7 +27,7 @@ def get_crawler(spidercls=None, settings_dict=None): """ from scrapy.crawler import CrawlerRunner from scrapy.settings import Settings - from scrapy.spider import Spider + from scrapy.spiders import Spider runner = CrawlerRunner(Settings(settings_dict)) return runner._create_crawler(spidercls or Spider) diff --git a/tests/spiders.py b/tests/spiders.py index c2956d741..516062929 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -5,7 +5,7 @@ Some spiders used for testing and benchmarking import time from six.moves.urllib.parse import urlencode -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request from scrapy.item import Item from scrapy.linkextractors import LinkExtractor diff --git a/tests/test_commands.py b/tests/test_commands.py index b95d0b0cc..7c10faf0c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -158,7 +158,7 @@ class MySpider(scrapy.Spider): fname = abspath(join(tmpdir, 'myspider.py')) with open(fname, 'w') as f: f.write(""" -from scrapy.spider import Spider +from scrapy.spiders import Spider """) p = self.proc('runspider', fname) log = p.stderr.read() diff --git a/tests/test_contracts.py b/tests/test_contracts.py index d7732f55d..1cea2afb7 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -2,7 +2,7 @@ from unittest import TextTestResult from twisted.trial import unittest -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request from scrapy.item import Item, Field from scrapy.contracts import ContractsManager diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 62fc280ee..df038aa88 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -24,7 +24,7 @@ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request from scrapy.settings import Settings from scrapy import optional_features diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 282035f5c..257eab609 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -2,7 +2,7 @@ from twisted.trial.unittest import TestCase from twisted.python.failure import Failure from scrapy.http import Request, Response -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 11de6e22d..67c57778d 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,7 +1,7 @@ import unittest from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 7f491f258..996b8c388 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -2,7 +2,7 @@ from unittest import TestCase import re from scrapy.http import Response, Request -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.downloadermiddlewares.cookies import CookiesMiddleware diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index 7aca415ef..9143611fc 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -1,7 +1,7 @@ from unittest import TestCase, main from scrapy.http import Response, XmlResponse from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware -from scrapy.spider import Spider +from scrapy.spiders import Spider from tests import get_testdata from scrapy.utils.test import assert_samelines diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 26520a20a..75d8a1921 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -3,7 +3,7 @@ import six from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 282d10829..446a99f36 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -1,7 +1,7 @@ import unittest from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index cef65b336..c30fa97c6 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -2,7 +2,7 @@ import unittest from scrapy.http import Request from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware -from scrapy.spider import Spider +from scrapy.spiders import Spider class TestSpider(Spider): http_user = 'foo' diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index ac954cc15..47d057e3f 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -8,7 +8,7 @@ from contextlib import contextmanager import pytest from scrapy.http import Response, HtmlResponse, Request -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.settings import Settings from scrapy.exceptions import IgnoreRequest from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 98df6d608..a18994ef3 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -3,7 +3,7 @@ from unittest import TestCase from os.path import join, abspath, dirname from gzip import GzipFile -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware from tests import tests_datadir diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 90609879c..191664076 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -5,7 +5,7 @@ from twisted.trial.unittest import TestCase, SkipTest from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request -from scrapy.spider import Spider +from scrapy.spiders import Spider spider = Spider('foo') diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 3f299f258..7e88e71af 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,7 +1,7 @@ import unittest from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response, HtmlResponse from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 969452cfb..c0381e144 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -7,7 +7,7 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \ from scrapy import optional_features from scrapy.downloadermiddlewares.retry import RetryMiddleware from scrapy.xlib.tx import ResponseFailed -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 64f2d0786..fb46ccff6 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.http import Request, Response -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index ddbb8f3ac..741c8de76 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -1,6 +1,6 @@ from unittest import TestCase -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware from scrapy.utils.test import get_crawler diff --git a/tests/test_engine.py b/tests/test_engine.py index bbb94fd58..d7ad88abb 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -22,7 +22,7 @@ from scrapy import signals from scrapy.utils.test import get_crawler from scrapy.xlib.pydispatch import dispatcher from tests import tests_datadir -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor from scrapy.http import Request diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 8446fd646..ec42ef8ab 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -1,6 +1,6 @@ import unittest -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.item import Item, Field from scrapy.logformatter import LogFormatter diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 24ba9d64a..7217eee90 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -6,7 +6,7 @@ from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.utils.request import request_fingerprint from scrapy.pipelines.media import MediaPipeline from scrapy.utils.signal import disconnect_all diff --git a/tests/test_spider.py b/tests/test_spider.py index f771399ca..f2dfd2dce 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -7,11 +7,10 @@ from testfixtures import LogCapture from twisted.trial import unittest from scrapy import signals -from scrapy.spider import Spider, BaseSpider from scrapy.settings import Settings from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse from scrapy.spiders.init import InitSpider -from scrapy.spiders import CrawlSpider, Rule, XMLFeedSpider, \ +from scrapy.spiders import Spider, BaseSpider, CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning @@ -116,7 +115,7 @@ class SpiderTest(unittest.TestCase): def test_log(self): spider = self.spider_class('example.com') - with mock.patch('scrapy.spider.Spider.logger') as mock_logger: + with mock.patch('scrapy.spiders.Spider.logger') as mock_logger: spider.log('test log msg', 'INFO') mock_logger.log.assert_called_once_with('INFO', 'test log msg') diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 42f2f29b3..7cb5e299b 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -6,7 +6,7 @@ from zope.interface.verify import verifyObject from twisted.trial import unittest -# ugly hack to avoid cyclic imports of scrapy.spider when running this test +# ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone from scrapy.interfaces import ISpiderLoader from scrapy.spiderloader import SpiderLoader diff --git a/tests/test_spiderloader/test_spiders/spider0.py b/tests/test_spiderloader/test_spiders/spider0.py index f1f19a1eb..75a90794e 100644 --- a/tests/test_spiderloader/test_spiders/spider0.py +++ b/tests/test_spiderloader/test_spiders/spider0.py @@ -1,4 +1,4 @@ -from scrapy.spider import Spider +from scrapy.spiders import Spider class Spider0(Spider): allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/tests/test_spiderloader/test_spiders/spider1.py b/tests/test_spiderloader/test_spiders/spider1.py index 16a533ca2..76efddc7f 100644 --- a/tests/test_spiderloader/test_spiders/spider1.py +++ b/tests/test_spiderloader/test_spiders/spider1.py @@ -1,4 +1,4 @@ -from scrapy.spider import Spider +from scrapy.spiders import Spider class Spider1(Spider): name = "spider1" diff --git a/tests/test_spiderloader/test_spiders/spider2.py b/tests/test_spiderloader/test_spiders/spider2.py index 4af6f7c41..0badd8437 100644 --- a/tests/test_spiderloader/test_spiders/spider2.py +++ b/tests/test_spiderloader/test_spiders/spider2.py @@ -1,4 +1,4 @@ -from scrapy.spider import Spider +from scrapy.spiders import Spider class Spider2(Spider): name = "spider2" diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py index b3e5f3da7..d406f2d4f 100644 --- a/tests/test_spiderloader/test_spiders/spider3.py +++ b/tests/test_spiderloader/test_spiders/spider3.py @@ -1,4 +1,4 @@ -from scrapy.spider import Spider +from scrapy.spiders import Spider class Spider3(Spider): name = "spider3" diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 5317795a3..a3cdc0114 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.http import Response, Request -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 5cd2c2566..a64400482 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -7,7 +7,7 @@ from twisted.internet import defer from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from scrapy.http import Response, Request -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 296e8b1a3..f88c806d7 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -3,7 +3,7 @@ from unittest import TestCase from six.moves.urllib.parse import urlparse from scrapy.http import Response, Request -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.spidermiddlewares.offsite import OffsiteMiddleware from scrapy.utils.test import get_crawler diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index f2815ebd3..d773ea8d3 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,7 +1,7 @@ from unittest import TestCase from scrapy.http import Response, Request -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.spidermiddlewares.referer import RefererMiddleware diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 1ef22ea07..dca868ecf 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request -from scrapy.spider import Spider +from scrapy.spiders import Spider class TestUrlLengthMiddleware(TestCase): diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 1ddce4b99..d83015bd9 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -3,7 +3,7 @@ from datetime import datetime from twisted.trial import unittest from scrapy.extensions.spiderstate import SpiderState -from scrapy.spider import Spider +from scrapy.spiders import Spider class SpiderStateTest(unittest.TestCase): diff --git a/tests/test_stats.py b/tests/test_stats.py index 34f72736c..5c7c0e6bb 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,6 +1,6 @@ import unittest -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.statscollectors import StatsCollector, DummyStatsCollector from scrapy.utils.test import get_crawler diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index 17cf82213..e9f220092 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -21,7 +21,7 @@ class ToplevelTestCase(TestCase): self.assertIs(scrapy.FormRequest, FormRequest) def test_spider_shortcut(self): - from scrapy.spider import Spider + from scrapy.spiders import Spider self.assertIs(scrapy.Spider, Spider) def test_selector_shortcut(self): diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 4ddc2f472..9139c0ad0 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -1,7 +1,7 @@ import unittest from scrapy.http import Request -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.utils.reqser import request_to_dict, request_from_dict class RequestSerializationTest(unittest.TestCase): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 959760068..860c76bae 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,6 +1,6 @@ import unittest -from scrapy.spider import Spider +from scrapy.spiders import Spider from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url __doctests__ = ['scrapy.utils.url']