Move scrapy/contrib/linkextractors to scrapy/linkextractors

docs/topics/firebug.rst

@@ -81,7 +81,7 @@ process and extract data from those pages.
 
 This is how the spider would look so far::
 
-   from scrapy.contrib.linkextractors import LinkExtractor
+   from scrapy.linkextractors import LinkExtractor
    from scrapy.contrib.spiders import CrawlSpider, Rule
 
    class GoogleDirectorySpider(CrawlSpider):

docs/topics/link-extractors.rst

@@ -8,7 +8,7 @@ Link extractors are objects whose only purpose is to extract links from web
 pages (:class:`scrapy.http.Response` objects) which will be eventually
 followed.
 
-There is ``scrapy.contrib.linkextractors import LinkExtractor`` available
+There is ``scrapy.linkextractors import LinkExtractor`` available
 in Scrapy, but you can create your own custom Link Extractors to suit your
 needs by implementing a simple interface.
 
@@ -30,16 +30,16 @@ extract links.
 Built-in link extractors reference
 ==================================
 
-.. module:: scrapy.contrib.linkextractors
+.. module:: scrapy.linkextractors
    :synopsis: Link extractors classes
 
 Link extractors classes bundled with Scrapy are provided in the
-:mod:`scrapy.contrib.linkextractors` module.
+:mod:`scrapy.linkextractors` module.
 
 The default link extractor is ``LinkExtractor``, which is the same as
 :class:`~.LxmlLinkExtractor`::
 
-    from scrapy.contrib.linkextractors import LinkExtractor
+    from scrapy.linkextractors import LinkExtractor
 
 There used to be other link extractor classes in previous Scrapy versions,
 but they are deprecated now.
@@ -47,7 +47,7 @@ but they are deprecated now.
 LxmlLinkExtractor
 -----------------
 
-.. module:: scrapy.contrib.linkextractors.lxmlhtml
+.. module:: scrapy.linkextractors.lxmlhtml
    :synopsis: lxml's HTMLParser-based link extractors
 
 

docs/topics/spiders.rst

@@ -395,7 +395,7 @@ Let's now take a look at an example CrawlSpider with rules::
 
     import scrapy
     from scrapy.contrib.spiders import CrawlSpider, Rule
-    from scrapy.contrib.linkextractors import LinkExtractor
+    from scrapy.linkextractors import LinkExtractor
 
     class MySpider(CrawlSpider):
         name = 'example.com'

scrapy/commands/bench.py

@@ -6,7 +6,7 @@ from six.moves.urllib.parse import urlencode
 
 import scrapy
 from scrapy.command import ScrapyCommand
-from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.linkextractors import LinkExtractor
 
 
 class Command(ScrapyCommand):

scrapy/link.py

@@ -1,7 +1,7 @@
 """
 This module defines the Link object used in Link extractors.
 
-For actual link extractors implementation see scrapy.contrib.linkextractor, or
+For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
 

scrapy/linkextractor.py

@@ -1,6 +1,6 @@
 """
 Common code and definitions used by Link extractors (located in
-scrapy.contrib.linkextractor).
+scrapy.linkextractors).
 """
 import re
 from six.moves.urllib.parse import urlparse

scrapy/linkextractors/__init__.py

@@ -1,5 +1,5 @@
 """
-scrapy.contrib.linkextractors
+scrapy.linkextractors
 
 This package contains a collection of Link Extractors.
 

scrapy/linkextractors/sgml.py

@@ -20,7 +20,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
     def __init__(self, tag="a", attr="href", unique=False, process_value=None):
         warnings.warn(
             "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
-            "Please use scrapy.contrib.linkextractors.LinkExtractor",
+            "Please use scrapy.linkextractors.LinkExtractor",
             ScrapyDeprecationWarning
         )
         SGMLParser.__init__(self)
@@ -103,7 +103,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
 
         warnings.warn(
             "SgmlLinkExtractor is deprecated and will be removed in future releases. "
-            "Please use scrapy.contrib.linkextractors.LinkExtractor",
+            "Please use scrapy.linkextractors.LinkExtractor",
             ScrapyDeprecationWarning
         )
 

scrapy/templates/spiders/crawl.tmpl

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import scrapy
-from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.linkextractors import LinkExtractor
 from scrapy.contrib.spiders import CrawlSpider, Rule
 
 from $project_name.items import ${ProjectName}Item

tests/py3-ignores.txt

@@ -5,7 +5,7 @@ tests/test_command_shell.py
 tests/test_commands.py
 tests/test_command_version.py
 tests/test_exporters.py
-tests/test_contrib_linkextractors.py
+tests/test_linkextractors.py
 tests/test_contrib_loader.py
 tests/test_crawl.py
 tests/test_crawler.py
@@ -88,6 +88,9 @@ scrapy/contrib/pipeline/files.py
 scrapy/contrib/linkextractors/sgml.py
 scrapy/contrib/linkextractors/regex.py
 scrapy/contrib/linkextractors/htmlparser.py
+scrapy/linkextractors/sgml.py
+scrapy/linkextractors/regex.py
+scrapy/linkextractors/htmlparser.py
 scrapy/contrib/downloadermiddleware/retry.py
 scrapy/contrib/downloadermiddleware/httpproxy.py
 scrapy/contrib/downloadermiddleware/cookies.py

tests/spiders.py

@@ -8,7 +8,7 @@ from six.moves.urllib.parse import urlencode
 from scrapy.spider import Spider
 from scrapy.http import Request
 from scrapy.item import Item
-from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.linkextractors import LinkExtractor
 
 
 class MetaSpider(Spider):

tests/test_engine.py

@@ -24,7 +24,7 @@ from scrapy.xlib.pydispatch import dispatcher
 from tests import tests_datadir
 from scrapy.spider import Spider
 from scrapy.item import Item, Field
-from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.linkextractors import LinkExtractor
 from scrapy.http import Request
 from scrapy.utils.signal import disconnect_all
 

tests/test_linkextractors.py

@@ -1,11 +1,11 @@
 import re
 import unittest
-from scrapy.contrib.linkextractors.regex import RegexLinkExtractor
+from scrapy.linkextractors.regex import RegexLinkExtractor
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.link import Link
-from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
-from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor
+from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
+from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
 from tests import get_testdata
 
 

tests/test_spider.py

@@ -13,7 +13,7 @@ from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlRespon
 from scrapy.contrib.spiders.init import InitSpider
 from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \
     CSVFeedSpider, SitemapSpider
-from scrapy.contrib.linkextractors import LinkExtractor
+from scrapy.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.test import get_crawler