diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index ad3f26b50..3bbb902b2 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -81,7 +81,7 @@ process and extract data from those pages. This is how the spider would look so far:: - from scrapy.contrib.linkextractors import LinkExtractor + from scrapy.linkextractors import LinkExtractor from scrapy.contrib.spiders import CrawlSpider, Rule class GoogleDirectorySpider(CrawlSpider): diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index f2f296fba..be3eb4537 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -8,7 +8,7 @@ Link extractors are objects whose only purpose is to extract links from web pages (:class:`scrapy.http.Response` objects) which will be eventually followed. -There is ``scrapy.contrib.linkextractors import LinkExtractor`` available +There is ``scrapy.linkextractors import LinkExtractor`` available in Scrapy, but you can create your own custom Link Extractors to suit your needs by implementing a simple interface. @@ -30,16 +30,16 @@ extract links. Built-in link extractors reference ================================== -.. module:: scrapy.contrib.linkextractors +.. module:: scrapy.linkextractors :synopsis: Link extractors classes Link extractors classes bundled with Scrapy are provided in the -:mod:`scrapy.contrib.linkextractors` module. +:mod:`scrapy.linkextractors` module. The default link extractor is ``LinkExtractor``, which is the same as :class:`~.LxmlLinkExtractor`:: - from scrapy.contrib.linkextractors import LinkExtractor + from scrapy.linkextractors import LinkExtractor There used to be other link extractor classes in previous Scrapy versions, but they are deprecated now. @@ -47,7 +47,7 @@ but they are deprecated now. LxmlLinkExtractor ----------------- -.. module:: scrapy.contrib.linkextractors.lxmlhtml +.. module:: scrapy.linkextractors.lxmlhtml :synopsis: lxml's HTMLParser-based link extractors diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 7c7d5d731..fdc5581a0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -395,7 +395,7 @@ Let's now take a look at an example CrawlSpider with rules:: import scrapy from scrapy.contrib.spiders import CrawlSpider, Rule - from scrapy.contrib.linkextractors import LinkExtractor + from scrapy.linkextractors import LinkExtractor class MySpider(CrawlSpider): name = 'example.com' diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 395597546..7c056a990 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -6,7 +6,7 @@ from six.moves.urllib.parse import urlencode import scrapy from scrapy.command import ScrapyCommand -from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.linkextractors import LinkExtractor class Command(ScrapyCommand): diff --git a/scrapy/link.py b/scrapy/link.py index 42c0e4f48..8bdcce761 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -1,7 +1,7 @@ """ This module defines the Link object used in Link extractors. -For actual link extractors implementation see scrapy.contrib.linkextractor, or +For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py index 227d79b46..2a4d18877 100644 --- a/scrapy/linkextractor.py +++ b/scrapy/linkextractor.py @@ -1,6 +1,6 @@ """ Common code and definitions used by Link extractors (located in -scrapy.contrib.linkextractor). +scrapy.linkextractors). """ import re from six.moves.urllib.parse import urlparse diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py similarity index 85% rename from scrapy/contrib/linkextractors/__init__.py rename to scrapy/linkextractors/__init__.py index 48b9c757a..28afaa2f1 100644 --- a/scrapy/contrib/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -1,5 +1,5 @@ """ -scrapy.contrib.linkextractors +scrapy.linkextractors This package contains a collection of Link Extractors. diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py similarity index 100% rename from scrapy/contrib/linkextractors/htmlparser.py rename to scrapy/linkextractors/htmlparser.py diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py similarity index 100% rename from scrapy/contrib/linkextractors/lxmlhtml.py rename to scrapy/linkextractors/lxmlhtml.py diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/linkextractors/regex.py similarity index 100% rename from scrapy/contrib/linkextractors/regex.py rename to scrapy/linkextractors/regex.py diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py similarity index 97% rename from scrapy/contrib/linkextractors/sgml.py rename to scrapy/linkextractors/sgml.py index 335773db1..b1f3da416 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -20,7 +20,7 @@ class BaseSgmlLinkExtractor(SGMLParser): def __init__(self, tag="a", attr="href", unique=False, process_value=None): warnings.warn( "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. " - "Please use scrapy.contrib.linkextractors.LinkExtractor", + "Please use scrapy.linkextractors.LinkExtractor", ScrapyDeprecationWarning ) SGMLParser.__init__(self) @@ -103,7 +103,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " - "Please use scrapy.contrib.linkextractors.LinkExtractor", + "Please use scrapy.linkextractors.LinkExtractor", ScrapyDeprecationWarning ) diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 0482a5496..b84e785fd 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import scrapy -from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.linkextractors import LinkExtractor from scrapy.contrib.spiders import CrawlSpider, Rule from $project_name.items import ${ProjectName}Item diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 8e5c5dc9b..3f1da0751 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -5,7 +5,7 @@ tests/test_command_shell.py tests/test_commands.py tests/test_command_version.py tests/test_exporters.py -tests/test_contrib_linkextractors.py +tests/test_linkextractors.py tests/test_contrib_loader.py tests/test_crawl.py tests/test_crawler.py @@ -88,6 +88,9 @@ scrapy/contrib/pipeline/files.py scrapy/contrib/linkextractors/sgml.py scrapy/contrib/linkextractors/regex.py scrapy/contrib/linkextractors/htmlparser.py +scrapy/linkextractors/sgml.py +scrapy/linkextractors/regex.py +scrapy/linkextractors/htmlparser.py scrapy/contrib/downloadermiddleware/retry.py scrapy/contrib/downloadermiddleware/httpproxy.py scrapy/contrib/downloadermiddleware/cookies.py diff --git a/tests/spiders.py b/tests/spiders.py index 5484fc5b9..c2956d741 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -8,7 +8,7 @@ from six.moves.urllib.parse import urlencode from scrapy.spider import Spider from scrapy.http import Request from scrapy.item import Item -from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.linkextractors import LinkExtractor class MetaSpider(Spider): diff --git a/tests/test_engine.py b/tests/test_engine.py index 04fae02c0..bbb94fd58 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -24,7 +24,7 @@ from scrapy.xlib.pydispatch import dispatcher from tests import tests_datadir from scrapy.spider import Spider from scrapy.item import Item, Field -from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.linkextractors import LinkExtractor from scrapy.http import Request from scrapy.utils.signal import disconnect_all diff --git a/tests/test_contrib_linkextractors.py b/tests/test_linkextractors.py similarity index 98% rename from tests/test_contrib_linkextractors.py rename to tests/test_linkextractors.py index a624f9e66..948289f8f 100644 --- a/tests/test_contrib_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,11 +1,11 @@ import re import unittest -from scrapy.contrib.linkextractors.regex import RegexLinkExtractor +from scrapy.linkextractors.regex import RegexLinkExtractor from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link -from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor -from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor -from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor +from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor +from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor +from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor from tests import get_testdata diff --git a/tests/test_spider.py b/tests/test_spider.py index 517fc0995..02352ec29 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -13,7 +13,7 @@ from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlRespon from scrapy.contrib.spiders.init import InitSpider from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider -from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref from scrapy.utils.test import get_crawler