diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 479b17f99..219616587 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -511,7 +511,7 @@ using a `trick to pass additional data to the callbacks .. note:: As an example spider that leverages this mechanism, check out the - :class:`~scrapy.contrib.spiders.CrawlSpider` class for a generic spider + :class:`~scrapy.spiders.CrawlSpider` class for a generic spider that implements a small rules engine that you can use to write your crawlers on top of it. diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 638de5712..f1f79ce70 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -399,7 +399,7 @@ Supported options: * ``--pipelines``: process items through pipelines -* ``--rules`` or ``-r``: use :class:`~scrapy.contrib.spiders.CrawlSpider` +* ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` rules to discover the callback (i.e. spider method) to use for parsing the response diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index be19bc44f..bb969eca3 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -288,7 +288,7 @@ HttpAuthMiddleware Example:: - from scrapy.contrib.spiders import CrawlSpider + from scrapy.spiders import CrawlSpider class SomeIntranetSiteSpider(CrawlSpider): diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 3bbb902b2..359c99450 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -74,15 +74,15 @@ So, based on that regular expression we can create the first crawling rule:: follow=True, ), -The :class:`~scrapy.contrib.spiders.Rule` object instructs -:class:`~scrapy.contrib.spiders.CrawlSpider` based spiders how to follow the +The :class:`~scrapy.spiders.Rule` object instructs +:class:`~scrapy.spiders.CrawlSpider` based spiders how to follow the category links. ``parse_category`` will be a method of the spider which will process and extract data from those pages. This is how the spider would look so far:: from scrapy.linkextractors import LinkExtractor - from scrapy.contrib.spiders import CrawlSpider, Rule + from scrapy.spiders import CrawlSpider, Rule class GoogleDirectorySpider(CrawlSpider): name = 'directory.google.com' diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index be3eb4537..edb047c86 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -18,10 +18,10 @@ of :class:`scrapy.link.Link` objects. Link extractors are meant to be instantiated once and their ``extract_links`` method called several times with different responses to extract links to follow. -Link extractors are used in the :class:`~scrapy.contrib.spiders.CrawlSpider` +Link extractors are used in the :class:`~scrapy.spiders.CrawlSpider` class (available in Scrapy), through a set of rules, but you can also use it in your spiders, even if you don't subclass from -:class:`~scrapy.contrib.spiders.CrawlSpider`, as its purpose is very simple: to +:class:`~scrapy.spiders.CrawlSpider`, as its purpose is very simple: to extract links. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index df030170c..d2fdd61b8 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -319,7 +319,7 @@ with a ``TestItem`` declared in a ``myproject.items`` module:: description = scrapy.Field() -.. module:: scrapy.contrib.spiders +.. module:: scrapy.spiders :synopsis: Collection of generic spiders CrawlSpider @@ -394,7 +394,7 @@ CrawlSpider example Let's now take a look at an example CrawlSpider with rules:: import scrapy - from scrapy.contrib.spiders import CrawlSpider, Rule + from scrapy.spiders import CrawlSpider, Rule from scrapy.linkextractors import LinkExtractor class MySpider(CrawlSpider): @@ -515,7 +515,7 @@ XMLFeedSpider example These spiders are pretty easy to use, let's have a look at one example:: - from scrapy.contrib.spiders import XMLFeedSpider + from scrapy.spiders import XMLFeedSpider from myproject.items import TestItem class MySpider(XMLFeedSpider): @@ -575,7 +575,7 @@ CSVFeedSpider example Let's see an example similar to the previous one, but using a :class:`CSVFeedSpider`:: - from scrapy.contrib.spiders import CSVFeedSpider + from scrapy.spiders import CSVFeedSpider from myproject.items import TestItem class MySpider(CSVFeedSpider): @@ -669,7 +669,7 @@ SitemapSpider examples Simplest example: process all urls discovered through sitemaps using the ``parse`` callback:: - from scrapy.contrib.spiders import SitemapSpider + from scrapy.spiders import SitemapSpider class MySpider(SitemapSpider): sitemap_urls = ['http://www.example.com/sitemap.xml'] @@ -680,7 +680,7 @@ Simplest example: process all urls discovered through sitemaps using the Process some urls with certain callback and other urls with a different callback:: - from scrapy.contrib.spiders import SitemapSpider + from scrapy.spiders import SitemapSpider class MySpider(SitemapSpider): sitemap_urls = ['http://www.example.com/sitemap.xml'] @@ -698,7 +698,7 @@ callback:: Follow sitemaps defined in the `robots.txt`_ file and only follow sitemaps whose url contains ``/sitemap_shop``:: - from scrapy.contrib.spiders import SitemapSpider + from scrapy.spiders import SitemapSpider class MySpider(SitemapSpider): sitemap_urls = ['http://www.example.com/robots.txt'] @@ -712,7 +712,7 @@ whose url contains ``/sitemap_shop``:: Combine SitemapSpider with other sources of urls:: - from scrapy.contrib.spiders import SitemapSpider + from scrapy.spiders import SitemapSpider class MySpider(SitemapSpider): sitemap_urls = ['http://www.example.com/robots.txt'] diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py deleted file mode 100644 index c16bb6c0a..000000000 --- a/scrapy/contrib/spiders/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from scrapy.contrib.spiders.crawl import CrawlSpider, Rule -from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider -from scrapy.contrib.spiders.sitemap import SitemapSpider diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py new file mode 100644 index 000000000..de4f90d51 --- /dev/null +++ b/scrapy/spiders/__init__.py @@ -0,0 +1,3 @@ +from scrapy.spiders.crawl import CrawlSpider, Rule +from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider +from scrapy.spiders.sitemap import SitemapSpider diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/spiders/crawl.py similarity index 100% rename from scrapy/contrib/spiders/crawl.py rename to scrapy/spiders/crawl.py diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/spiders/feed.py similarity index 100% rename from scrapy/contrib/spiders/feed.py rename to scrapy/spiders/feed.py diff --git a/scrapy/contrib/spiders/init.py b/scrapy/spiders/init.py similarity index 100% rename from scrapy/contrib/spiders/init.py rename to scrapy/spiders/init.py diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/spiders/sitemap.py similarity index 100% rename from scrapy/contrib/spiders/sitemap.py rename to scrapy/spiders/sitemap.py diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index b84e785fd..a179d16ff 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import scrapy from scrapy.linkextractors import LinkExtractor -from scrapy.contrib.spiders import CrawlSpider, Rule +from scrapy.spiders import CrawlSpider, Rule from $project_name.items import ${ProjectName}Item diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 7e10cd8a8..69c606538 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from scrapy.contrib.spiders import CSVFeedSpider +from scrapy.spiders import CSVFeedSpider from $project_name.items import ${ProjectName}Item diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index a0e26aa14..9c0910d23 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from scrapy.contrib.spiders import XMLFeedSpider +from scrapy.spiders import XMLFeedSpider from $project_name.items import ${ProjectName}Item diff --git a/tests/test_spider.py b/tests/test_spider.py index 02352ec29..f771399ca 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -10,8 +10,8 @@ from scrapy import signals from scrapy.spider import Spider, BaseSpider from scrapy.settings import Settings from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse -from scrapy.contrib.spiders.init import InitSpider -from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \ +from scrapy.spiders.init import InitSpider +from scrapy.spiders import CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 334ec00f3..045e72117 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -3,7 +3,7 @@ from scrapy.http import Request from scrapy.item import BaseItem from scrapy.utils.spider import iterate_spider_output, iter_spider_classes -from scrapy.contrib.spiders import CrawlSpider +from scrapy.spiders import CrawlSpider class MyBaseSpider(CrawlSpider):