diff --git a/docs/faq.rst b/docs/faq.rst index 80d258b55..d98b06056 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,7 +149,7 @@ middleware (enabled by default) whose purpose is to filter out requests to domains outside the ones covered by the spider. For more info see: -:class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware`. +:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware`. What is the recommended way to deploy a Scrapy crawler in production? --------------------------------------------------------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3e68ad925..a049f1438 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -906,11 +906,11 @@ SPIDER_MIDDLEWARES_BASE Default:: { - 'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50, - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, - 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, - 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800, - 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900, + 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500, + 'scrapy.spidermiddlewares.referer.RefererMiddleware': 700, + 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800, + 'scrapy.spidermiddlewares.depth.DepthMiddleware': 900, } A dict containing the spider middlewares enabled by default in Scrapy. You @@ -1001,7 +1001,7 @@ URLLENGTH_LIMIT Default: ``2083`` -Scope: ``contrib.spidermiddleware.urllength`` +Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. For more information about the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index abeae2bce..6e82333f5 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -43,7 +43,7 @@ value. For example, if you want to disable the off-site middleware:: SPIDER_MIDDLEWARES = { 'myproject.middlewares.CustomSpiderMiddleware': 543, - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': None, + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -55,7 +55,7 @@ Writing your own spider middleware Each middleware component is a Python class that defines one or more of the following methods: -.. module:: scrapy.contrib.spidermiddleware +.. module:: scrapy.spidermiddlewares .. class:: SpiderMiddleware @@ -178,7 +178,7 @@ For a list of the components enabled by default (and their orders) see the DepthMiddleware --------------- -.. module:: scrapy.contrib.spidermiddleware.depth +.. module:: scrapy.spidermiddlewares.depth :synopsis: Depth Spider Middleware .. class:: DepthMiddleware @@ -199,7 +199,7 @@ DepthMiddleware HttpErrorMiddleware ------------------- -.. module:: scrapy.contrib.spidermiddleware.httperror +.. module:: scrapy.spidermiddlewares.httperror :synopsis: HTTP Error Spider Middleware .. class:: HttpErrorMiddleware @@ -264,7 +264,7 @@ Pass all responses, regardless of its status code. OffsiteMiddleware ----------------- -.. module:: scrapy.contrib.spidermiddleware.offsite +.. module:: scrapy.spidermiddlewares.offsite :synopsis: Offsite Spider Middleware .. class:: OffsiteMiddleware @@ -298,7 +298,7 @@ OffsiteMiddleware RefererMiddleware ----------------- -.. module:: scrapy.contrib.spidermiddleware.referer +.. module:: scrapy.spidermiddlewares.referer :synopsis: Referer Spider Middleware .. class:: RefererMiddleware @@ -323,7 +323,7 @@ Whether to enable referer middleware. UrlLengthMiddleware ------------------- -.. module:: scrapy.contrib.spidermiddleware.urllength +.. module:: scrapy.spidermiddlewares.urllength :synopsis: URL Length Spider Middleware .. class:: UrlLengthMiddleware diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index fdc5581a0..df030170c 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -77,7 +77,7 @@ scrapy.Spider An optional list of strings containing domains that this spider is allowed to crawl. Requests for URLs not belonging to the domain names specified in this list won't be followed if - :class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware` is enabled. + :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. .. attribute:: start_urls diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f4dcdc1a2..a7fe36648 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -224,11 +224,11 @@ SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side - 'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50, - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, - 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, - 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800, - 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900, + 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500, + 'scrapy.spidermiddlewares.referer.RefererMiddleware': 700, + 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800, + 'scrapy.spidermiddlewares.depth.DepthMiddleware': 900, # Spider side } diff --git a/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/spidermiddlewares/__init__.py similarity index 100% rename from scrapy/contrib/spidermiddleware/__init__.py rename to scrapy/spidermiddlewares/__init__.py diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/spidermiddlewares/depth.py similarity index 100% rename from scrapy/contrib/spidermiddleware/depth.py rename to scrapy/spidermiddlewares/depth.py diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/spidermiddlewares/httperror.py similarity index 100% rename from scrapy/contrib/spidermiddleware/httperror.py rename to scrapy/spidermiddlewares/httperror.py diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/spidermiddlewares/offsite.py similarity index 100% rename from scrapy/contrib/spidermiddleware/offsite.py rename to scrapy/spidermiddlewares/offsite.py diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/spidermiddlewares/referer.py similarity index 100% rename from scrapy/contrib/spidermiddleware/referer.py rename to scrapy/spidermiddlewares/referer.py diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/spidermiddlewares/urllength.py similarity index 100% rename from scrapy/contrib/spidermiddleware/urllength.py rename to scrapy/spidermiddlewares/urllength.py diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index e7ae75ed2..8120f1a95 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,6 +1,6 @@ from unittest import TestCase -from scrapy.contrib.spidermiddleware.depth import DepthMiddleware +from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.http import Response, Request from scrapy.spider import Spider from scrapy.statscol import StatsCollector @@ -37,7 +37,7 @@ class TestDepthMiddleware(TestCase): rdm = self.stats.get_value('request_depth_max', spider=self.spider) self.assertEquals(rdm, 1) - + def tearDown(self): self.stats.close_spider(self.spider, '') diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 503865f63..5cd2c2566 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -8,7 +8,7 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from scrapy.http import Response, Request from scrapy.spider import Spider -from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpError +from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index e5e99002a..296e8b1a3 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -4,7 +4,7 @@ from six.moves.urllib.parse import urlparse from scrapy.http import Response, Request from scrapy.spider import Spider -from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware +from scrapy.spidermiddlewares.offsite import OffsiteMiddleware from scrapy.utils.test import get_crawler class TestOffsiteMiddleware(TestCase): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index f408719d2..f2815ebd3 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.http import Response, Request from scrapy.spider import Spider -from scrapy.contrib.spidermiddleware.referer import RefererMiddleware +from scrapy.spidermiddlewares.referer import RefererMiddleware class TestRefererMiddleware(TestCase): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index ca2de18f2..1ef22ea07 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,6 +1,6 @@ from unittest import TestCase -from scrapy.contrib.spidermiddleware.urllength import UrlLengthMiddleware +from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spider import Spider