From d7c444fefbd9b69698618d3d18b0c3e4858ec3e5 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 20 Apr 2015 21:23:05 -0300 Subject: [PATCH 01/25] Move scrapy/contrib/downloadermiddleware to scrapy/downloadermiddlewares --- docs/faq.rst | 4 +-- docs/topics/downloader-middleware.rst | 32 +++++++++---------- docs/topics/settings.rst | 32 +++++++++---------- .../downloadermiddleware/decompression.py | 4 +-- .../__init__.py | 0 .../ajaxcrawl.py | 0 .../chunked.py | 0 .../cookies.py | 0 .../decompression.py | 0 .../defaultheaders.py | 0 .../downloadtimeout.py | 0 .../httpauth.py | 0 .../httpcache.py | 0 .../httpcompression.py | 0 .../httpproxy.py | 0 .../redirect.py | 0 .../retry.py | 0 .../robotstxt.py | 0 .../stats.py | 0 .../useragent.py | 0 scrapy/settings/default_settings.py | 30 ++++++++--------- scrapy/utils/misc.py | 2 +- tests/py3-ignores.txt | 3 ++ ...test_downloadermiddleware_ajaxcrawlable.py | 4 +-- tests/test_downloadermiddleware_cookies.py | 2 +- ...test_downloadermiddleware_decompression.py | 4 +-- ...est_downloadermiddleware_defaultheaders.py | 2 +- ...st_downloadermiddleware_downloadtimeout.py | 2 +- tests/test_downloadermiddleware_httpauth.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 2 +- ...st_downloadermiddleware_httpcompression.py | 2 +- tests/test_downloadermiddleware_httpproxy.py | 2 +- tests/test_downloadermiddleware_redirect.py | 2 +- tests/test_downloadermiddleware_retry.py | 2 +- tests/test_downloadermiddleware_robotstxt.py | 2 +- tests/test_downloadermiddleware_stats.py | 4 +-- tests/test_downloadermiddleware_useragent.py | 2 +- 37 files changed, 72 insertions(+), 69 deletions(-) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/__init__.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/ajaxcrawl.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/chunked.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/cookies.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/decompression.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/defaultheaders.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/downloadtimeout.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/httpauth.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/httpcache.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/httpcompression.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/httpproxy.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/redirect.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/retry.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/robotstxt.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/stats.py (100%) rename scrapy/{contrib/downloadermiddleware => downloadermiddlewares}/useragent.py (100%) diff --git a/docs/faq.rst b/docs/faq.rst index b92d173dd..f593e4f16 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -64,7 +64,7 @@ Does Scrapy work with HTTP proxies? Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP Proxy downloader middleware. See -:class:`~scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware`. +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`. How can I scrape an item with attributes in different pages? ------------------------------------------------------------ @@ -113,7 +113,7 @@ See previous question. Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- -Yes, see :class:`~scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware`. +Yes, see :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`. Why does Scrapy download pages in English instead of my native language? ------------------------------------------------------------------------ diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3d24b29ad..be19bc44f 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -42,7 +42,7 @@ as its value. For example, if you want to disable the user-agent middleware:: DOWNLOADER_MIDDLEWARES = { 'myproject.middlewares.CustomDownloaderMiddleware': 543, - 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': None, + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -54,7 +54,7 @@ Writing your own downloader middleware Each middleware component is a Python class that defines one or more of the following methods: -.. module:: scrapy.contrib.downloadermiddleware +.. module:: scrapy.downloadermiddlewares .. class:: DownloaderMiddleware @@ -169,7 +169,7 @@ For a list of the components enabled by default (and their orders) see the CookiesMiddleware ----------------- -.. module:: scrapy.contrib.downloadermiddleware.cookies +.. module:: scrapy.downloadermiddlewares.cookies :synopsis: Cookies Downloader Middleware .. class:: CookiesMiddleware @@ -246,7 +246,7 @@ Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: DefaultHeadersMiddleware ------------------------ -.. module:: scrapy.contrib.downloadermiddleware.defaultheaders +.. module:: scrapy.downloadermiddlewares.defaultheaders :synopsis: Default Headers Downloader Middleware .. class:: DefaultHeadersMiddleware @@ -257,7 +257,7 @@ DefaultHeadersMiddleware DownloadTimeoutMiddleware ------------------------- -.. module:: scrapy.contrib.downloadermiddleware.downloadtimeout +.. module:: scrapy.downloadermiddlewares.downloadtimeout :synopsis: Download timeout middleware .. class:: DownloadTimeoutMiddleware @@ -275,7 +275,7 @@ DownloadTimeoutMiddleware HttpAuthMiddleware ------------------ -.. module:: scrapy.contrib.downloadermiddleware.httpauth +.. module:: scrapy.downloadermiddlewares.httpauth :synopsis: HTTP Auth downloader middleware .. class:: HttpAuthMiddleware @@ -304,7 +304,7 @@ HttpAuthMiddleware HttpCacheMiddleware ------------------- -.. module:: scrapy.contrib.downloadermiddleware.httpcache +.. module:: scrapy.downloadermiddlewares.httpcache :synopsis: HTTP Cache downloader middleware .. class:: HttpCacheMiddleware @@ -579,7 +579,7 @@ This setting is specific to the Filesystem backend. HttpCompressionMiddleware ------------------------- -.. module:: scrapy.contrib.downloadermiddleware.httpcompression +.. module:: scrapy.downloadermiddlewares.httpcompression :synopsis: Http Compression Middleware .. class:: HttpCompressionMiddleware @@ -603,7 +603,7 @@ Whether the Compression middleware will be enabled. ChunkedTransferMiddleware ------------------------- -.. module:: scrapy.contrib.downloadermiddleware.chunked +.. module:: scrapy.downloadermiddlewares.chunked :synopsis: Chunked Transfer Middleware .. class:: ChunkedTransferMiddleware @@ -613,7 +613,7 @@ ChunkedTransferMiddleware HttpProxyMiddleware ------------------- -.. module:: scrapy.contrib.downloadermiddleware.httpproxy +.. module:: scrapy.downloadermiddlewares.httpproxy :synopsis: Http Proxy Middleware .. versionadded:: 0.8 @@ -641,7 +641,7 @@ HttpProxyMiddleware RedirectMiddleware ------------------ -.. module:: scrapy.contrib.downloadermiddleware.redirect +.. module:: scrapy.downloadermiddlewares.redirect :synopsis: Redirection Middleware .. class:: RedirectMiddleware @@ -731,7 +731,7 @@ The maximum meta-refresh delay (in seconds) to follow the redirection. RetryMiddleware --------------- -.. module:: scrapy.contrib.downloadermiddleware.retry +.. module:: scrapy.downloadermiddlewares.retry :synopsis: Retry Middleware .. class:: RetryMiddleware @@ -800,7 +800,7 @@ connections lost, etc) are always retried. RobotsTxtMiddleware ------------------- -.. module:: scrapy.contrib.downloadermiddleware.robotstxt +.. module:: scrapy.downloadermiddlewares.robotstxt :synopsis: robots.txt middleware .. class:: RobotsTxtMiddleware @@ -828,7 +828,7 @@ the request will be ignored by this middleware even if DownloaderStats --------------- -.. module:: scrapy.contrib.downloadermiddleware.stats +.. module:: scrapy.downloadermiddlewares.stats :synopsis: Downloader Stats Middleware .. class:: DownloaderStats @@ -842,7 +842,7 @@ DownloaderStats UserAgentMiddleware ------------------- -.. module:: scrapy.contrib.downloadermiddleware.useragent +.. module:: scrapy.downloadermiddlewares.useragent :synopsis: User Agent Middleware .. class:: UserAgentMiddleware @@ -857,7 +857,7 @@ UserAgentMiddleware AjaxCrawlMiddleware ------------------- -.. module:: scrapy.contrib.downloadermiddleware.ajaxcrawl +.. module:: scrapy.downloadermiddlewares.ajaxcrawl .. class:: AjaxCrawlMiddleware diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3936c04b8..3e68ad925 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -239,7 +239,7 @@ Default:: } The default headers used for Scrapy HTTP Requests. They're populated in the -:class:`~scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware`. +:class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`. .. setting:: DEPTH_LIMIT @@ -335,20 +335,20 @@ DOWNLOADER_MIDDLEWARES_BASE Default:: { - 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300, - 'scrapy.contrib.downloadermiddleware.downloadtimeout.DownloadTimeoutMiddleware': 350, - 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, - 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, - 'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550, - 'scrapy.contrib.downloadermiddleware.redirect.MetaRefreshMiddleware': 580, - 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 590, - 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, - 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, - 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830, - 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, - 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, + 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, + 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400, + 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550, + 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580, + 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590, + 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, + 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, + 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, + 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830, + 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, } A dict containing the downloader middlewares enabled by default in Scrapy. You @@ -837,7 +837,7 @@ ROBOTSTXT_OBEY Default: ``False`` -Scope: ``scrapy.contrib.downloadermiddleware.robotstxt`` +Scope: ``scrapy.downloadermiddlewares.robotstxt`` If enabled, Scrapy will respect robots.txt policies. For more information see :ref:`topics-dlmw-robots` diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py index 8b1d61b3d..1f8490587 100644 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ b/scrapy/contrib_exp/downloadermiddleware/decompression.py @@ -1,7 +1,7 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning warnings.warn("Module `scrapy.contrib_exp.downloadermiddleware.decompression` is deprecated, " - "use `scrapy.contrib.downloadermiddleware.decompression` instead", + "use `scrapy.downloadermiddlewares.decompression` instead", ScrapyDeprecationWarning, stacklevel=2) -from scrapy.contrib.downloadermiddleware.decompression import DecompressionMiddleware +from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware diff --git a/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/downloadermiddlewares/__init__.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/__init__.py rename to scrapy/downloadermiddlewares/__init__.py diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/ajaxcrawl.py rename to scrapy/downloadermiddlewares/ajaxcrawl.py diff --git a/scrapy/contrib/downloadermiddleware/chunked.py b/scrapy/downloadermiddlewares/chunked.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/chunked.py rename to scrapy/downloadermiddlewares/chunked.py diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/downloadermiddlewares/cookies.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/cookies.py rename to scrapy/downloadermiddlewares/cookies.py diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/downloadermiddlewares/decompression.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/decompression.py rename to scrapy/downloadermiddlewares/decompression.py diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/defaultheaders.py rename to scrapy/downloadermiddlewares/defaultheaders.py diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/downloadtimeout.py rename to scrapy/downloadermiddlewares/downloadtimeout.py diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/httpauth.py rename to scrapy/downloadermiddlewares/httpauth.py diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/httpcache.py rename to scrapy/downloadermiddlewares/httpcache.py diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/httpcompression.py rename to scrapy/downloadermiddlewares/httpcompression.py diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/httpproxy.py rename to scrapy/downloadermiddlewares/httpproxy.py diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/downloadermiddlewares/redirect.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/redirect.py rename to scrapy/downloadermiddlewares/redirect.py diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/downloadermiddlewares/retry.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/retry.py rename to scrapy/downloadermiddlewares/retry.py diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/robotstxt.py rename to scrapy/downloadermiddlewares/robotstxt.py diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/downloadermiddlewares/stats.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/stats.py rename to scrapy/downloadermiddlewares/stats.py diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/downloadermiddlewares/useragent.py similarity index 100% rename from scrapy/contrib/downloadermiddleware/useragent.py rename to scrapy/downloadermiddlewares/useragent.py diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 0c3d7c5bd..3323386aa 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -80,21 +80,21 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side - 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300, - 'scrapy.contrib.downloadermiddleware.downloadtimeout.DownloadTimeoutMiddleware': 350, - 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, - 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, - 'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550, - 'scrapy.contrib.downloadermiddleware.ajaxcrawl.AjaxCrawlMiddleware': 560, - 'scrapy.contrib.downloadermiddleware.redirect.MetaRefreshMiddleware': 580, - 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 590, - 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, - 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, - 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830, - 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, - 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, + 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, + 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400, + 'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500, + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550, + 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560, + 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580, + 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590, + 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, + 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, + 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, + 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830, + 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, # Downloader side } diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 3152db6c7..f1aa4fcdb 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -32,7 +32,7 @@ def load_object(path): """Load an object given its absolute object path, and return it. object can be a class, function, variable o instance. - path ie: 'scrapy.contrib.downloadermiddelware.redirect.RedirectMiddleware' + path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' """ try: diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index d27d5dd3c..cf814c15c 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -91,6 +91,9 @@ scrapy/contrib/linkextractors/htmlparser.py scrapy/contrib/downloadermiddleware/retry.py scrapy/contrib/downloadermiddleware/httpproxy.py scrapy/contrib/downloadermiddleware/cookies.py +scrapy/downloadermiddlewares/retry.py +scrapy/downloadermiddlewares/httpproxy.py +scrapy/downloadermiddlewares/cookies.py scrapy/contrib/statsmailer.py scrapy/contrib/memusage.py scrapy/commands/deploy.py diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index e73e62538..11de6e22d 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,11 +1,11 @@ import unittest -from scrapy.contrib.downloadermiddleware.ajaxcrawl import AjaxCrawlMiddleware +from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware from scrapy.spider import Spider from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler -__doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawl'] +__doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl'] class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 3b1853c82..7f491f258 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -3,7 +3,7 @@ import re from scrapy.http import Response, Request from scrapy.spider import Spider -from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware +from scrapy.downloadermiddlewares.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index f3f862604..7aca415ef 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -1,6 +1,6 @@ from unittest import TestCase, main from scrapy.http import Response, XmlResponse -from scrapy.contrib.downloadermiddleware.decompression import DecompressionMiddleware +from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware from scrapy.spider import Spider from tests import get_testdata from scrapy.utils.test import assert_samelines @@ -16,7 +16,7 @@ def _test_data(formats): class DecompressionMiddlewareTest(TestCase): - + test_formats = ['tar', 'xml.bz2', 'xml.gz', 'zip'] uncompressed_body, test_responses = _test_data(test_formats) diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 09973b367..26520a20a 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -1,7 +1,7 @@ from unittest import TestCase import six -from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware +from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spider import Spider from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index aba06686a..282d10829 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -1,6 +1,6 @@ import unittest -from scrapy.contrib.downloadermiddleware.downloadtimeout import DownloadTimeoutMiddleware +from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware from scrapy.spider import Spider from scrapy.http import Request from scrapy.utils.test import get_crawler diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index adfcd802d..cef65b336 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,7 +1,7 @@ import unittest from scrapy.http import Request -from scrapy.contrib.downloadermiddleware.httpauth import HttpAuthMiddleware +from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spider import Spider class TestSpider(Spider): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 663d67be0..9c1678488 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -12,7 +12,7 @@ from scrapy.spider import Spider from scrapy.settings import Settings from scrapy.exceptions import IgnoreRequest from scrapy.utils.test import get_crawler -from scrapy.contrib.downloadermiddleware.httpcache import HttpCacheMiddleware +from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware class _BaseTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 1cc6f44c1..98df6d608 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -5,7 +5,7 @@ from gzip import GzipFile from scrapy.spider import Spider from scrapy.http import Response, Request, HtmlResponse -from scrapy.contrib.downloadermiddleware.httpcompression import HttpCompressionMiddleware +from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware from tests import tests_datadir from w3lib.encoding import resolve_encoding diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 58825c6cf..90609879c 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -2,7 +2,7 @@ import os import sys from twisted.trial.unittest import TestCase, SkipTest -from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware +from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.spider import Spider diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index fb70b13ae..3f299f258 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,6 +1,6 @@ import unittest -from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware, MetaRefreshMiddleware +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware from scrapy.spider import Spider from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response, HtmlResponse diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 669abea05..969452cfb 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -5,7 +5,7 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionLost, TCPTimedOutError from scrapy import optional_features -from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware +from scrapy.downloadermiddlewares.retry import RetryMiddleware from scrapy.xlib.tx import ResponseFailed from scrapy.spider import Spider from scrapy.http import Request, Response diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 8b7ac7a6b..bc245dcec 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -3,7 +3,7 @@ import re from twisted.internet import reactor from twisted.internet.defer import Deferred from twisted.trial import unittest -from scrapy.contrib.downloadermiddleware.robotstxt import RobotsTxtMiddleware +from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.settings import Settings diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index b790ff09a..64f2d0786 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,6 +1,6 @@ from unittest import TestCase -from scrapy.contrib.downloadermiddleware.stats import DownloaderStats +from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spider import Spider from scrapy.utils.test import get_crawler @@ -22,7 +22,7 @@ class TestDownloaderStats(TestCase): self.mw.process_request(self.req, self.spider) self.assertEqual(self.crawler.stats.get_value('downloader/request_count', \ spider=self.spider), 1) - + def test_process_response(self): self.mw.process_response(self.req, self.res, self.spider) self.assertEqual(self.crawler.stats.get_value('downloader/response_count', \ diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 909d03ba5..ddbb8f3ac 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.spider import Spider from scrapy.http import Request -from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware +from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware from scrapy.utils.test import get_crawler From 6b4c00cc9bfc00e715bd8fe71f1e4b214e7575da Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 20 Apr 2015 22:15:41 -0300 Subject: [PATCH 02/25] scrapy/contrib/downloadermiddleware shims --- scrapy/contrib/downloadermiddleware/__init__.py | 0 scrapy/contrib/downloadermiddleware/ajaxcrawl.py | 7 +++++++ scrapy/contrib/downloadermiddleware/chunked.py | 7 +++++++ scrapy/contrib/downloadermiddleware/cookies.py | 7 +++++++ scrapy/contrib/downloadermiddleware/decompression.py | 7 +++++++ scrapy/contrib/downloadermiddleware/defaultheaders.py | 7 +++++++ scrapy/contrib/downloadermiddleware/downloadtimeout.py | 7 +++++++ scrapy/contrib/downloadermiddleware/httpauth.py | 7 +++++++ scrapy/contrib/downloadermiddleware/httpcache.py | 7 +++++++ scrapy/contrib/downloadermiddleware/httpcompression.py | 7 +++++++ scrapy/contrib/downloadermiddleware/httpproxy.py | 7 +++++++ scrapy/contrib/downloadermiddleware/redirect.py | 7 +++++++ scrapy/contrib/downloadermiddleware/retry.py | 7 +++++++ scrapy/contrib/downloadermiddleware/robotstxt.py | 7 +++++++ scrapy/contrib/downloadermiddleware/stats.py | 7 +++++++ scrapy/contrib/downloadermiddleware/useragent.py | 7 +++++++ 16 files changed, 105 insertions(+) create mode 100644 scrapy/contrib/downloadermiddleware/__init__.py create mode 100644 scrapy/contrib/downloadermiddleware/ajaxcrawl.py create mode 100644 scrapy/contrib/downloadermiddleware/chunked.py create mode 100644 scrapy/contrib/downloadermiddleware/cookies.py create mode 100644 scrapy/contrib/downloadermiddleware/decompression.py create mode 100644 scrapy/contrib/downloadermiddleware/defaultheaders.py create mode 100644 scrapy/contrib/downloadermiddleware/downloadtimeout.py create mode 100644 scrapy/contrib/downloadermiddleware/httpauth.py create mode 100644 scrapy/contrib/downloadermiddleware/httpcache.py create mode 100644 scrapy/contrib/downloadermiddleware/httpcompression.py create mode 100644 scrapy/contrib/downloadermiddleware/httpproxy.py create mode 100644 scrapy/contrib/downloadermiddleware/redirect.py create mode 100644 scrapy/contrib/downloadermiddleware/retry.py create mode 100644 scrapy/contrib/downloadermiddleware/robotstxt.py create mode 100644 scrapy/contrib/downloadermiddleware/stats.py create mode 100644 scrapy/contrib/downloadermiddleware/useragent.py diff --git a/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/contrib/downloadermiddleware/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py new file mode 100644 index 000000000..90ebc46b6 --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.ajaxcrawl` is deprecated, " + "use `scrapy.downloadermiddlewares.ajaxcrawl` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.ajaxcrawl import * diff --git a/scrapy/contrib/downloadermiddleware/chunked.py b/scrapy/contrib/downloadermiddleware/chunked.py new file mode 100644 index 000000000..1322c9083 --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/chunked.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.chunked` is deprecated, " + "use `scrapy.downloadermiddlewares.chunked` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.chunked import * diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py new file mode 100644 index 000000000..bad970690 --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/cookies.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.cookies` is deprecated, " + "use `scrapy.downloadermiddlewares.cookies` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.cookies import * diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/contrib/downloadermiddleware/decompression.py new file mode 100644 index 000000000..a541aa61e --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/decompression.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.decompression` is deprecated, " + "use `scrapy.downloadermiddlewares.decompression` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.decompression import * diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py new file mode 100644 index 000000000..cf023dc8f --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/defaultheaders.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.defaultheaders` is deprecated, " + "use `scrapy.downloadermiddlewares.defaultheaders` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.defaultheaders import * diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py new file mode 100644 index 000000000..84bd06acf --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/downloadtimeout.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.downloadtimeout` is deprecated, " + "use `scrapy.downloadermiddlewares.downloadtimeout` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.downloadtimeout import * diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py new file mode 100644 index 000000000..a37ffa0dc --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/httpauth.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, " + "use `scrapy.downloadermiddlewares.httpauth` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.httpauth import * diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py new file mode 100644 index 000000000..f5f068204 --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcache` is deprecated, " + "use `scrapy.downloadermiddlewares.httpcache` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.httpcache import * diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py new file mode 100644 index 000000000..8a52ec50b --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/httpcompression.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcompression` is deprecated, " + "use `scrapy.downloadermiddlewares.httpcompression` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.httpcompression import * diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py new file mode 100644 index 000000000..d94d85076 --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/httpproxy.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpproxy` is deprecated, " + "use `scrapy.downloadermiddlewares.httpproxy` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.httpproxy import * diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py new file mode 100644 index 000000000..824eee8ae --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.redirect` is deprecated, " + "use `scrapy.downloadermiddlewares.redirect` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.redirect import * diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py new file mode 100644 index 000000000..aafe0f508 --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.retry` is deprecated, " + "use `scrapy.downloadermiddlewares.retry` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.retry import * diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py new file mode 100644 index 000000000..408f760a0 --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.robotstxt` is deprecated, " + "use `scrapy.downloadermiddlewares.robotstxt` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.robotstxt import * diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py new file mode 100644 index 000000000..fa84a8206 --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/stats.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.stats` is deprecated, " + "use `scrapy.downloadermiddlewares.stats` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.stats import * diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/contrib/downloadermiddleware/useragent.py new file mode 100644 index 000000000..893d5241c --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/useragent.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.downloadermiddleware.useragent` is deprecated, " + "use `scrapy.downloadermiddlewares.useragent` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.downloadermiddlewares.useragent import * From 7804b3d778902d93b844f4adf59f650c6b61ef00 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 20 Apr 2015 22:41:06 -0300 Subject: [PATCH 03/25] Move scrapy/contrib/exporter to scrapy/exporters --- docs/faq.rst | 2 +- docs/topics/exporters.rst | 6 ++--- docs/topics/feed-exports.rst | 22 +++++++++---------- .../exporter => exporters}/__init__.py | 0 scrapy/settings/default_settings.py | 14 ++++++------ tests/py3-ignores.txt | 2 +- ..._contrib_exporter.py => test_exporters.py} | 2 +- 7 files changed, 24 insertions(+), 24 deletions(-) rename scrapy/{contrib/exporter => exporters}/__init__.py (100%) rename tests/{test_contrib_exporter.py => test_exporters.py} (99%) diff --git a/docs/faq.rst b/docs/faq.rst index f593e4f16..80d258b55 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -160,7 +160,7 @@ Can I use JSON for large exports? --------------------------------- It'll depend on how large your output is. See :ref:`this warning -` in :class:`~scrapy.contrib.exporter.JsonItemExporter` +` in :class:`~scrapy.exporters.JsonItemExporter` documentation. Can I return (Twisted) deferreds from signal handlers? diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 43846852b..af469eb7b 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -4,7 +4,7 @@ Item Exporters ============== -.. module:: scrapy.contrib.exporter +.. module:: scrapy.exporters :synopsis: Item Exporters Once you have scraped your items, you often want to persist or export those @@ -40,7 +40,7 @@ Here you can see an :doc:`Item Pipeline ` which uses an Item Exporter to export scraped items to different files, one per spider:: from scrapy import signals - from scrapy.contrib.exporter import XmlItemExporter + from scrapy.exporters import XmlItemExporter class XmlExportPipeline(object): @@ -117,7 +117,7 @@ after your custom code. Example:: - from scrapy.contrib.exporter import XmlItemExporter + from scrapy.exporter import XmlItemExporter class ProductXmlExporter(XmlItemExporter): diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 8166a7a4e..27d601a19 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -37,7 +37,7 @@ JSON ---- * :setting:`FEED_FORMAT`: ``json`` - * Exporter used: :class:`~scrapy.contrib.exporter.JsonItemExporter` + * Exporter used: :class:`~scrapy.exporters.JsonItemExporter` * See :ref:`this warning ` if you're using JSON with large feeds. @@ -47,7 +47,7 @@ JSON lines ---------- * :setting:`FEED_FORMAT`: ``jsonlines`` - * Exporter used: :class:`~scrapy.contrib.exporter.JsonLinesItemExporter` + * Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter` .. _topics-feed-format-csv: @@ -55,7 +55,7 @@ CSV --- * :setting:`FEED_FORMAT`: ``csv`` - * Exporter used: :class:`~scrapy.contrib.exporter.CsvItemExporter` + * Exporter used: :class:`~scrapy.exporters.CsvItemExporter` * To specify columns to export and their order use :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this option, but it is important for CSV because unlike many other export @@ -67,7 +67,7 @@ XML --- * :setting:`FEED_FORMAT`: ``xml`` - * Exporter used: :class:`~scrapy.contrib.exporter.XmlItemExporter` + * Exporter used: :class:`~scrapy.exporters.XmlItemExporter` .. _topics-feed-format-pickle: @@ -75,7 +75,7 @@ Pickle ------ * :setting:`FEED_FORMAT`: ``pickle`` - * Exporter used: :class:`~scrapy.contrib.exporter.PickleItemExporter` + * Exporter used: :class:`~scrapy.exporters.PickleItemExporter` .. _topics-feed-format-marshal: @@ -83,7 +83,7 @@ Marshal ------- * :setting:`FEED_FORMAT`: ``marshal`` - * Exporter used: :class:`~scrapy.contrib.exporter.MarshalItemExporter` + * Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` .. _topics-feed-storage: @@ -300,11 +300,11 @@ FEED_EXPORTERS_BASE Default:: FEED_EXPORTERS_BASE = { - 'json': 'scrapy.contrib.exporter.JsonItemExporter', - 'jsonlines': 'scrapy.contrib.exporter.JsonLinesItemExporter', - 'csv': 'scrapy.contrib.exporter.CsvItemExporter', - 'xml': 'scrapy.contrib.exporter.XmlItemExporter', - 'marshal': 'scrapy.contrib.exporter.MarshalItemExporter', + 'json': 'scrapy.exporters.JsonItemExporter', + 'jsonlines': 'scrapy.exporters.JsonLinesItemExporter', + 'csv': 'scrapy.exporters.CsvItemExporter', + 'xml': 'scrapy.exporters.XmlItemExporter', + 'marshal': 'scrapy.exporters.MarshalItemExporter', } A dict containing the built-in feed exporters supported by Scrapy. diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/exporters/__init__.py similarity index 100% rename from scrapy/contrib/exporter/__init__.py rename to scrapy/exporters/__init__.py diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3323386aa..17db550a7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -139,13 +139,13 @@ FEED_STORAGES_BASE = { } FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { - 'json': 'scrapy.contrib.exporter.JsonItemExporter', - 'jsonlines': 'scrapy.contrib.exporter.JsonLinesItemExporter', - 'jl': 'scrapy.contrib.exporter.JsonLinesItemExporter', - 'csv': 'scrapy.contrib.exporter.CsvItemExporter', - 'xml': 'scrapy.contrib.exporter.XmlItemExporter', - 'marshal': 'scrapy.contrib.exporter.MarshalItemExporter', - 'pickle': 'scrapy.contrib.exporter.PickleItemExporter', + 'json': 'scrapy.exporters.JsonItemExporter', + 'jsonlines': 'scrapy.exporters.JsonLinesItemExporter', + 'jl': 'scrapy.exporters.JsonLinesItemExporter', + 'csv': 'scrapy.exporters.CsvItemExporter', + 'xml': 'scrapy.exporters.XmlItemExporter', + 'marshal': 'scrapy.exporters.MarshalItemExporter', + 'pickle': 'scrapy.exporters.PickleItemExporter', } HTTPCACHE_ENABLED = False diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index cf814c15c..8e5c5dc9b 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -4,7 +4,7 @@ tests/test_command_fetch.py tests/test_command_shell.py tests/test_commands.py tests/test_command_version.py -tests/test_contrib_exporter.py +tests/test_exporters.py tests/test_contrib_linkextractors.py tests/test_contrib_loader.py tests/test_crawl.py diff --git a/tests/test_contrib_exporter.py b/tests/test_exporters.py similarity index 99% rename from tests/test_contrib_exporter.py rename to tests/test_exporters.py index 746aeb65b..df1f8f4d5 100644 --- a/tests/test_contrib_exporter.py +++ b/tests/test_exporters.py @@ -9,7 +9,7 @@ import lxml.etree from scrapy.item import Item, Field from scrapy.utils.python import str_to_unicode -from scrapy.contrib.exporter import ( +from scrapy.exporters import ( BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, PythonItemExporter ) From 152594ce990e0f5923d87f968e354c1a17bf463d Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 20 Apr 2015 22:46:23 -0300 Subject: [PATCH 04/25] scrapy/contrib/exporter shims --- scrapy/contrib/exporter/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 scrapy/contrib/exporter/__init__.py diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py new file mode 100644 index 000000000..d434a7bfd --- /dev/null +++ b/scrapy/contrib/exporter/__init__.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.exporter` is deprecated, " + "use `scrapy.exporters` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.exporters import * From cf064b143747f4fd7e48670f478d40fa2101bb01 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 20 Apr 2015 22:55:33 -0300 Subject: [PATCH 05/25] Move scrapy/contrib/linkextractors to scrapy/linkextractors --- docs/topics/firebug.rst | 2 +- docs/topics/link-extractors.rst | 10 +++++----- docs/topics/spiders.rst | 2 +- scrapy/commands/bench.py | 2 +- scrapy/link.py | 2 +- scrapy/linkextractor.py | 2 +- scrapy/{contrib => }/linkextractors/__init__.py | 2 +- scrapy/{contrib => }/linkextractors/htmlparser.py | 0 scrapy/{contrib => }/linkextractors/lxmlhtml.py | 0 scrapy/{contrib => }/linkextractors/regex.py | 0 scrapy/{contrib => }/linkextractors/sgml.py | 4 ++-- scrapy/templates/spiders/crawl.tmpl | 2 +- tests/py3-ignores.txt | 5 ++++- tests/spiders.py | 2 +- tests/test_engine.py | 2 +- ...ontrib_linkextractors.py => test_linkextractors.py} | 8 ++++---- tests/test_spider.py | 2 +- 17 files changed, 25 insertions(+), 22 deletions(-) rename scrapy/{contrib => }/linkextractors/__init__.py (85%) rename scrapy/{contrib => }/linkextractors/htmlparser.py (100%) rename scrapy/{contrib => }/linkextractors/lxmlhtml.py (100%) rename scrapy/{contrib => }/linkextractors/regex.py (100%) rename scrapy/{contrib => }/linkextractors/sgml.py (97%) rename tests/{test_contrib_linkextractors.py => test_linkextractors.py} (98%) diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index ad3f26b50..3bbb902b2 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -81,7 +81,7 @@ process and extract data from those pages. This is how the spider would look so far:: - from scrapy.contrib.linkextractors import LinkExtractor + from scrapy.linkextractors import LinkExtractor from scrapy.contrib.spiders import CrawlSpider, Rule class GoogleDirectorySpider(CrawlSpider): diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index f2f296fba..be3eb4537 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -8,7 +8,7 @@ Link extractors are objects whose only purpose is to extract links from web pages (:class:`scrapy.http.Response` objects) which will be eventually followed. -There is ``scrapy.contrib.linkextractors import LinkExtractor`` available +There is ``scrapy.linkextractors import LinkExtractor`` available in Scrapy, but you can create your own custom Link Extractors to suit your needs by implementing a simple interface. @@ -30,16 +30,16 @@ extract links. Built-in link extractors reference ================================== -.. module:: scrapy.contrib.linkextractors +.. module:: scrapy.linkextractors :synopsis: Link extractors classes Link extractors classes bundled with Scrapy are provided in the -:mod:`scrapy.contrib.linkextractors` module. +:mod:`scrapy.linkextractors` module. The default link extractor is ``LinkExtractor``, which is the same as :class:`~.LxmlLinkExtractor`:: - from scrapy.contrib.linkextractors import LinkExtractor + from scrapy.linkextractors import LinkExtractor There used to be other link extractor classes in previous Scrapy versions, but they are deprecated now. @@ -47,7 +47,7 @@ but they are deprecated now. LxmlLinkExtractor ----------------- -.. module:: scrapy.contrib.linkextractors.lxmlhtml +.. module:: scrapy.linkextractors.lxmlhtml :synopsis: lxml's HTMLParser-based link extractors diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 7c7d5d731..fdc5581a0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -395,7 +395,7 @@ Let's now take a look at an example CrawlSpider with rules:: import scrapy from scrapy.contrib.spiders import CrawlSpider, Rule - from scrapy.contrib.linkextractors import LinkExtractor + from scrapy.linkextractors import LinkExtractor class MySpider(CrawlSpider): name = 'example.com' diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 395597546..7c056a990 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -6,7 +6,7 @@ from six.moves.urllib.parse import urlencode import scrapy from scrapy.command import ScrapyCommand -from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.linkextractors import LinkExtractor class Command(ScrapyCommand): diff --git a/scrapy/link.py b/scrapy/link.py index 42c0e4f48..8bdcce761 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -1,7 +1,7 @@ """ This module defines the Link object used in Link extractors. -For actual link extractors implementation see scrapy.contrib.linkextractor, or +For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py index 227d79b46..2a4d18877 100644 --- a/scrapy/linkextractor.py +++ b/scrapy/linkextractor.py @@ -1,6 +1,6 @@ """ Common code and definitions used by Link extractors (located in -scrapy.contrib.linkextractor). +scrapy.linkextractors). """ import re from six.moves.urllib.parse import urlparse diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py similarity index 85% rename from scrapy/contrib/linkextractors/__init__.py rename to scrapy/linkextractors/__init__.py index 48b9c757a..28afaa2f1 100644 --- a/scrapy/contrib/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -1,5 +1,5 @@ """ -scrapy.contrib.linkextractors +scrapy.linkextractors This package contains a collection of Link Extractors. diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py similarity index 100% rename from scrapy/contrib/linkextractors/htmlparser.py rename to scrapy/linkextractors/htmlparser.py diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py similarity index 100% rename from scrapy/contrib/linkextractors/lxmlhtml.py rename to scrapy/linkextractors/lxmlhtml.py diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/linkextractors/regex.py similarity index 100% rename from scrapy/contrib/linkextractors/regex.py rename to scrapy/linkextractors/regex.py diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py similarity index 97% rename from scrapy/contrib/linkextractors/sgml.py rename to scrapy/linkextractors/sgml.py index 335773db1..b1f3da416 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -20,7 +20,7 @@ class BaseSgmlLinkExtractor(SGMLParser): def __init__(self, tag="a", attr="href", unique=False, process_value=None): warnings.warn( "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. " - "Please use scrapy.contrib.linkextractors.LinkExtractor", + "Please use scrapy.linkextractors.LinkExtractor", ScrapyDeprecationWarning ) SGMLParser.__init__(self) @@ -103,7 +103,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " - "Please use scrapy.contrib.linkextractors.LinkExtractor", + "Please use scrapy.linkextractors.LinkExtractor", ScrapyDeprecationWarning ) diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 0482a5496..b84e785fd 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import scrapy -from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.linkextractors import LinkExtractor from scrapy.contrib.spiders import CrawlSpider, Rule from $project_name.items import ${ProjectName}Item diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 8e5c5dc9b..3f1da0751 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -5,7 +5,7 @@ tests/test_command_shell.py tests/test_commands.py tests/test_command_version.py tests/test_exporters.py -tests/test_contrib_linkextractors.py +tests/test_linkextractors.py tests/test_contrib_loader.py tests/test_crawl.py tests/test_crawler.py @@ -88,6 +88,9 @@ scrapy/contrib/pipeline/files.py scrapy/contrib/linkextractors/sgml.py scrapy/contrib/linkextractors/regex.py scrapy/contrib/linkextractors/htmlparser.py +scrapy/linkextractors/sgml.py +scrapy/linkextractors/regex.py +scrapy/linkextractors/htmlparser.py scrapy/contrib/downloadermiddleware/retry.py scrapy/contrib/downloadermiddleware/httpproxy.py scrapy/contrib/downloadermiddleware/cookies.py diff --git a/tests/spiders.py b/tests/spiders.py index 5484fc5b9..c2956d741 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -8,7 +8,7 @@ from six.moves.urllib.parse import urlencode from scrapy.spider import Spider from scrapy.http import Request from scrapy.item import Item -from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.linkextractors import LinkExtractor class MetaSpider(Spider): diff --git a/tests/test_engine.py b/tests/test_engine.py index 04fae02c0..bbb94fd58 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -24,7 +24,7 @@ from scrapy.xlib.pydispatch import dispatcher from tests import tests_datadir from scrapy.spider import Spider from scrapy.item import Item, Field -from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.linkextractors import LinkExtractor from scrapy.http import Request from scrapy.utils.signal import disconnect_all diff --git a/tests/test_contrib_linkextractors.py b/tests/test_linkextractors.py similarity index 98% rename from tests/test_contrib_linkextractors.py rename to tests/test_linkextractors.py index a624f9e66..948289f8f 100644 --- a/tests/test_contrib_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,11 +1,11 @@ import re import unittest -from scrapy.contrib.linkextractors.regex import RegexLinkExtractor +from scrapy.linkextractors.regex import RegexLinkExtractor from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link -from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor -from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor -from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor +from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor +from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor +from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor from tests import get_testdata diff --git a/tests/test_spider.py b/tests/test_spider.py index 517fc0995..02352ec29 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -13,7 +13,7 @@ from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlRespon from scrapy.contrib.spiders.init import InitSpider from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider -from scrapy.contrib.linkextractors import LinkExtractor +from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref from scrapy.utils.test import get_crawler From 569156be190fb7e86108c29001d6233a6698a510 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 20 Apr 2015 23:03:07 -0300 Subject: [PATCH 06/25] scrapy/contrib/linkextractors shims --- scrapy/contrib/linkextractors/__init__.py | 7 +++++++ scrapy/contrib/linkextractors/htmlparser.py | 7 +++++++ scrapy/contrib/linkextractors/lxmlhtml.py | 7 +++++++ scrapy/contrib/linkextractors/regex.py | 7 +++++++ scrapy/contrib/linkextractors/sgml.py | 7 +++++++ 5 files changed, 35 insertions(+) create mode 100644 scrapy/contrib/linkextractors/__init__.py create mode 100644 scrapy/contrib/linkextractors/htmlparser.py create mode 100644 scrapy/contrib/linkextractors/lxmlhtml.py create mode 100644 scrapy/contrib/linkextractors/regex.py create mode 100644 scrapy/contrib/linkextractors/sgml.py diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/contrib/linkextractors/__init__.py new file mode 100644 index 000000000..976658df3 --- /dev/null +++ b/scrapy/contrib/linkextractors/__init__.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.linkextractors` is deprecated, " + "use `scrapy.linkextractors` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.linkextractors import * diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py new file mode 100644 index 000000000..ff03da98f --- /dev/null +++ b/scrapy/contrib/linkextractors/htmlparser.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.linkextractors.htmlparser` is deprecated, " + "use `scrapy.linkextractors.htmlparser` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.linkextractors.htmlparser import * diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py new file mode 100644 index 000000000..fc2b7de3c --- /dev/null +++ b/scrapy/contrib/linkextractors/lxmlhtml.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.linkextractors.lxmlhtml` is deprecated, " + "use `scrapy.linkextractors.lxmlhtml` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.linkextractors.lxmlhtml import * diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py new file mode 100644 index 000000000..97bda29c1 --- /dev/null +++ b/scrapy/contrib/linkextractors/regex.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.linkextractors.regex` is deprecated, " + "use `scrapy.linkextractors.regex` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.linkextractors.regex import * diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py new file mode 100644 index 000000000..a5a598208 --- /dev/null +++ b/scrapy/contrib/linkextractors/sgml.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.linkextractors.sgml` is deprecated, " + "use `scrapy.linkextractors.sgml` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.linkextractors.sgml import * From b47228ada8ac583f123434686e8d3493505c7127 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 20 Apr 2015 23:21:41 -0300 Subject: [PATCH 07/25] Move scrapy/contrib/loader to scrapy/loader --- docs/contributing.rst | 8 ++--- docs/topics/loaders.rst | 32 +++++++++---------- scrapy/{contrib => }/loader/__init__.py | 0 scrapy/{contrib => }/loader/common.py | 0 scrapy/{contrib => }/loader/processor.py | 0 tests/py3-ignores.txt | 2 +- ...{test_contrib_loader.py => test_loader.py} | 4 +-- tests/test_squeue.py | 2 +- 8 files changed, 24 insertions(+), 24 deletions(-) rename scrapy/{contrib => }/loader/__init__.py (100%) rename scrapy/{contrib => }/loader/common.py (100%) rename scrapy/{contrib => }/loader/processor.py (100%) rename tests/{test_contrib_loader.py => test_loader.py} (99%) diff --git a/docs/contributing.rst b/docs/contributing.rst index f49bc536e..ad9a3805a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -150,9 +150,9 @@ To run all tests go to the root directory of Scrapy source code and run: ``tox`` -To run a specific test (say ``tests/test_contrib_loader.py``) use: +To run a specific test (say ``tests/test_loader.py``) use: - ``tox -- tests/test_contrib_loader.py`` + ``tox -- tests/test_loader.py`` Writing tests @@ -166,11 +166,11 @@ Scrapy uses unit-tests, which are located in the `tests/`_ directory. Their module name typically resembles the full path of the module they're testing. For example, the item loaders code is in:: - scrapy.contrib.loader + scrapy.loader And their unit-tests are in:: - tests/test_contrib_loader.py + tests/test_loader.py .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 603d28f78..a5efa3fb9 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -4,7 +4,7 @@ Item Loaders ============ -.. module:: scrapy.contrib.loader +.. module:: scrapy.loader :synopsis: Item Loader class Item Loaders provide a convenient mechanism for populating scraped :ref:`Items @@ -39,7 +39,7 @@ Here is a typical Item Loader usage in a :ref:`Spider `, using the :ref:`Product item ` declared in the :ref:`Items chapter `:: - from scrapy.contrib.loader import ItemLoader + from scrapy.loader import ItemLoader from myproject.items import Product def parse(self, response): @@ -150,8 +150,8 @@ Declaring Item Loaders Item Loaders are declared like Items, by using a class definition syntax. Here is an example:: - from scrapy.contrib.loader import ItemLoader - from scrapy.contrib.loader.processor import TakeFirst, MapCompose, Join + from scrapy.loader import ItemLoader + from scrapy.loader.processor import TakeFirst, MapCompose, Join class ProductLoader(ItemLoader): @@ -182,7 +182,7 @@ output processors to use: in the :ref:`Item Field ` metadata. Here is an example:: import scrapy - from scrapy.contrib.loader.processor import Join, MapCompose, TakeFirst + from scrapy.loader.processor import Join, MapCompose, TakeFirst from w3lib.html import remove_tags def filter_price(value): @@ -201,7 +201,7 @@ metadata. Here is an example:: :: - >>> from scrapy.contrib.loader import ItemLoader + >>> from scrapy.loader import ItemLoader >>> il = ItemLoader(item=Product()) >>> il.add_value('name', [u'Welcome to my', u'website']) >>> il.add_value('price', [u'€', u'1000']) @@ -309,7 +309,7 @@ ItemLoader objects Examples:: - >>> from scrapy.contrib.loader.processor import TakeFirst + >>> from scrapy.loader.processor import TakeFirst >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)') 'FOO` @@ -513,7 +513,7 @@ those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default Product Item Loader (``ProductLoader``):: - from scrapy.contrib.loader.processor import MapCompose + from scrapy.loader.processor import MapCompose from myproject.ItemLoaders import ProductLoader def strip_dashes(x): @@ -526,7 +526,7 @@ Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it:: - from scrapy.contrib.loader.processor import MapCompose + from scrapy.loader.processor import MapCompose from myproject.ItemLoaders import ProductLoader from myproject.utils.xml import remove_cdata @@ -551,7 +551,7 @@ needs. Available built-in processors ============================= -.. module:: scrapy.contrib.loader.processor +.. module:: scrapy.loader.processor :synopsis: A collection of processors to use with Item Loaders Even though you can use any callable function as input and output processors, @@ -570,7 +570,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.contrib.loader.processor import Identity + >>> from scrapy.loader.processor import Identity >>> proc = Identity() >>> proc(['one', 'two', 'three']) ['one', 'two', 'three'] @@ -583,7 +583,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.contrib.loader.processor import TakeFirst + >>> from scrapy.loader.processor import TakeFirst >>> proc = TakeFirst() >>> proc(['', 'one', 'two', 'three']) 'one' @@ -598,7 +598,7 @@ Here is a list of all built-in processors: Examples:: - >>> from scrapy.contrib.loader.processor import Join + >>> from scrapy.loader.processor import Join >>> proc = Join() >>> proc(['one', 'two', 'three']) u'one two three' @@ -619,7 +619,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.contrib.loader.processor import Compose + >>> from scrapy.loader.processor import Compose >>> proc = Compose(lambda v: v[0], str.upper) >>> proc(['hello', 'world']) 'HELLO' @@ -666,7 +666,7 @@ Here is a list of all built-in processors: >>> def filter_world(x): ... return None if x == 'world' else x ... - >>> from scrapy.contrib.loader.processor import MapCompose + >>> from scrapy.loader.processor import MapCompose >>> proc = MapCompose(filter_world, unicode.upper) >>> proc([u'hello', u'world', u'this', u'is', u'scrapy']) [u'HELLO, u'THIS', u'IS', u'SCRAPY'] @@ -683,7 +683,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.contrib.loader.processor import SelectJmes, Compose, MapCompose + >>> from scrapy.loader.processor import SelectJmes, Compose, MapCompose >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries >>> proc({'foo': 'bar'}) 'bar' diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/loader/__init__.py similarity index 100% rename from scrapy/contrib/loader/__init__.py rename to scrapy/loader/__init__.py diff --git a/scrapy/contrib/loader/common.py b/scrapy/loader/common.py similarity index 100% rename from scrapy/contrib/loader/common.py rename to scrapy/loader/common.py diff --git a/scrapy/contrib/loader/processor.py b/scrapy/loader/processor.py similarity index 100% rename from scrapy/contrib/loader/processor.py rename to scrapy/loader/processor.py diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 3f1da0751..69df2f7a2 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -6,7 +6,7 @@ tests/test_commands.py tests/test_command_version.py tests/test_exporters.py tests/test_linkextractors.py -tests/test_contrib_loader.py +tests/test_loader.py tests/test_crawl.py tests/test_crawler.py tests/test_downloader_handlers.py diff --git a/tests/test_contrib_loader.py b/tests/test_loader.py similarity index 99% rename from tests/test_contrib_loader.py rename to tests/test_loader.py index fd65e8200..c51092ad0 100644 --- a/tests/test_contrib_loader.py +++ b/tests/test_loader.py @@ -1,8 +1,8 @@ import unittest from functools import partial -from scrapy.contrib.loader import ItemLoader -from scrapy.contrib.loader.processor import Join, Identity, TakeFirst, \ +from scrapy.loader import ItemLoader +from scrapy.loader.processor import Join, Identity, TakeFirst, \ Compose, MapCompose, SelectJmes from scrapy.item import Item, Field from scrapy.selector import Selector diff --git a/tests/test_squeue.py b/tests/test_squeue.py index 83ffcc4b7..16014a897 100644 --- a/tests/test_squeue.py +++ b/tests/test_squeue.py @@ -2,7 +2,7 @@ from queuelib.tests import test_queue as t from scrapy.squeue import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue from scrapy.item import Item, Field from scrapy.http import Request -from scrapy.contrib.loader import ItemLoader +from scrapy.loader import ItemLoader class TestItem(Item): name = Field() From d7e60f3c718bb9fbfd4151825b009a0a1d042fc4 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 20 Apr 2015 23:27:41 -0300 Subject: [PATCH 08/25] scrapy/contrib/loader shims --- scrapy/contrib/loader/__init__.py | 7 +++++++ scrapy/contrib/loader/common.py | 7 +++++++ scrapy/contrib/loader/processor.py | 7 +++++++ 3 files changed, 21 insertions(+) create mode 100644 scrapy/contrib/loader/__init__.py create mode 100644 scrapy/contrib/loader/common.py create mode 100644 scrapy/contrib/loader/processor.py diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py new file mode 100644 index 000000000..2b9453e18 --- /dev/null +++ b/scrapy/contrib/loader/__init__.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.loader` is deprecated, " + "use `scrapy.loader` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.loader import * diff --git a/scrapy/contrib/loader/common.py b/scrapy/contrib/loader/common.py new file mode 100644 index 000000000..a59b2b7b1 --- /dev/null +++ b/scrapy/contrib/loader/common.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.loader.common` is deprecated, " + "use `scrapy.loader.common` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.loader.common import * diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py new file mode 100644 index 000000000..68ddff4d4 --- /dev/null +++ b/scrapy/contrib/loader/processor.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.loader.processor` is deprecated, " + "use `scrapy.loader.processor` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.loader.processor import * From 8021df18d4f1703ad6b40c6e42e08aa627d629ae Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 20 Apr 2015 23:43:38 -0300 Subject: [PATCH 09/25] Move scrapy/contrib/pipeline to scrapy/pipelines --- docs/topics/media-pipeline.rst | 12 ++++++------ scrapy/{contrib/pipeline => pipelines}/__init__.py | 0 scrapy/{contrib/pipeline => pipelines}/files.py | 2 +- scrapy/{contrib/pipeline => pipelines}/images.py | 4 ++-- scrapy/{contrib/pipeline => pipelines}/media.py | 0 scrapy/settings/default_settings.py | 2 +- tests/py3-ignores.txt | 2 ++ tests/test_pipeline_files.py | 2 +- tests/test_pipeline_images.py | 2 +- tests/test_pipeline_media.py | 2 +- 10 files changed, 15 insertions(+), 13 deletions(-) rename scrapy/{contrib/pipeline => pipelines}/__init__.py (100%) rename scrapy/{contrib/pipeline => pipelines}/files.py (99%) rename scrapy/{contrib/pipeline => pipelines}/images.py (97%) rename scrapy/{contrib/pipeline => pipelines}/media.py (100%) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 4b88c60fd..c18ed25ae 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -4,7 +4,7 @@ Downloading and processing files and images =========================================== -.. currentmodule:: scrapy.contrib.pipeline.images +.. currentmodule:: scrapy.pipelines.images Scrapy provides reusable :doc:`item pipelines ` for downloading fies attached to a particular item (for example, when you scrape @@ -114,11 +114,11 @@ To enable your media pipeline you must first add it to your project For Images Pipeline, use:: - ITEM_PIPELINES = {'scrapy.contrib.pipeline.images.ImagesPipeline': 1} + ITEM_PIPELINES = {'scrapy.pipelines.images.ImagesPipeline': 1} For Files Pipeline, use:: - ITEM_PIPELINES = {'scrapy.contrib.pipeline.files.FilesPipeline': 1} + ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1} .. note:: @@ -258,7 +258,7 @@ By default, there are no size constraints, so all images are processed. Extending the Media Pipelines ============================= -.. module:: scrapy.contrib.pipeline.files +.. module:: scrapy.pipelines.files :synopsis: Files Pipeline See here the methods that you can override in your custom Files Pipeline: @@ -338,7 +338,7 @@ See here the methods that you can override in your custom Files Pipeline: By default, the :meth:`item_completed` method returns the item. -.. module:: scrapy.contrib.pipeline.images +.. module:: scrapy.pipelines.images :synopsis: Images Pipeline See here the methods that you can override in your custom Images Pipeline: @@ -374,7 +374,7 @@ Here is a full example of the Images Pipeline whose methods are examplified above:: import scrapy - from scrapy.contrib.pipeline.images import ImagesPipeline + from scrapy.pipelines.images import ImagesPipeline from scrapy.exceptions import DropItem class MyImagesPipeline(ImagesPipeline): diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/pipelines/__init__.py similarity index 100% rename from scrapy/contrib/pipeline/__init__.py rename to scrapy/pipelines/__init__.py diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/pipelines/files.py similarity index 99% rename from scrapy/contrib/pipeline/files.py rename to scrapy/pipelines/files.py index 4809005d1..2b17e0f0a 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/pipelines/files.py @@ -21,7 +21,7 @@ except ImportError: from twisted.internet import defer, threads -from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.pipelines.media import MediaPipeline from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.misc import md5sum diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/pipelines/images.py similarity index 97% rename from scrapy/contrib/pipeline/images.py rename to scrapy/pipelines/images.py index 3cbfed49b..8b3bc2222 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/pipelines/images.py @@ -17,8 +17,8 @@ from PIL import Image from scrapy.utils.misc import md5sum from scrapy.http import Request from scrapy.exceptions import DropItem -#TODO: from scrapy.contrib.pipeline.media import MediaPipeline -from scrapy.contrib.pipeline.files import FileException, FilesPipeline +#TODO: from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.files import FileException, FilesPipeline class NoimagesDrop(DropItem): diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/pipelines/media.py similarity index 100% rename from scrapy/contrib/pipeline/media.py rename to scrapy/pipelines/media.py diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 17db550a7..f4dcdc1a2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,7 +159,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.DummyPolicy' HTTPCACHE_GZIP = False -ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' +ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} ITEM_PIPELINES_BASE = {} diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 69df2f7a2..a29b419b0 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -85,6 +85,8 @@ scrapy/core/downloader/handlers/ftp.py scrapy/core/downloader/webclient.py scrapy/contrib/pipeline/images.py scrapy/contrib/pipeline/files.py +scrapy/pipelines/images.py +scrapy/pipelines/files.py scrapy/contrib/linkextractors/sgml.py scrapy/contrib/linkextractors/regex.py scrapy/contrib/linkextractors/htmlparser.py diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 84fe4927d..b12f41174 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -8,7 +8,7 @@ from shutil import rmtree from twisted.trial import unittest from twisted.internet import defer -from scrapy.contrib.pipeline.files import FilesPipeline, FSFilesStore +from scrapy.pipelines.files import FilesPipeline, FSFilesStore from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index f5750b4fc..04cec4b8e 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -9,7 +9,7 @@ from twisted.trial import unittest from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings -from scrapy.contrib.pipeline.images import ImagesPipeline +from scrapy.pipelines.images import ImagesPipeline skip = False try: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 0e946303b..24ba9d64a 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -8,7 +8,7 @@ from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response from scrapy.spider import Spider from scrapy.utils.request import request_fingerprint -from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.pipelines.media import MediaPipeline from scrapy.utils.signal import disconnect_all from scrapy import signals From c97a69c9071677b0ff7974e8fd1ef42acb9e207a Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Mon, 20 Apr 2015 23:47:30 -0300 Subject: [PATCH 10/25] scrapy/contrib/pipeline shims --- scrapy/contrib/pipeline/__init__.py | 7 +++++++ scrapy/contrib/pipeline/files.py | 7 +++++++ scrapy/contrib/pipeline/images.py | 7 +++++++ scrapy/contrib/pipeline/media.py | 7 +++++++ 4 files changed, 28 insertions(+) create mode 100644 scrapy/contrib/pipeline/__init__.py create mode 100644 scrapy/contrib/pipeline/files.py create mode 100644 scrapy/contrib/pipeline/images.py create mode 100644 scrapy/contrib/pipeline/media.py diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/contrib/pipeline/__init__.py new file mode 100644 index 000000000..aedf34a3f --- /dev/null +++ b/scrapy/contrib/pipeline/__init__.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.pipeline` is deprecated, " + "use `scrapy.pipelines` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.pipelines import * diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py new file mode 100644 index 000000000..cd1238b5d --- /dev/null +++ b/scrapy/contrib/pipeline/files.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.pipeline.files` is deprecated, " + "use `scrapy.pipelines.files` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.pipelines.files import * diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py new file mode 100644 index 000000000..4f5ce4c40 --- /dev/null +++ b/scrapy/contrib/pipeline/images.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.pipeline.images` is deprecated, " + "use `scrapy.pipelines.images` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.pipelines.images import * diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py new file mode 100644 index 000000000..4b4fea560 --- /dev/null +++ b/scrapy/contrib/pipeline/media.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.pipeline.media` is deprecated, " + "use `scrapy.pipelines.media` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.pipelines.media import * From 180272c09272c2dd47af65d5fb4cb914b051623a Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 21 Apr 2015 13:07:24 -0300 Subject: [PATCH 11/25] Move scrapy/contrib/spidermiddleware to scrapy/spidermiddlewares --- docs/faq.rst | 2 +- docs/topics/settings.rst | 12 ++++++------ docs/topics/spider-middleware.rst | 14 +++++++------- docs/topics/spiders.rst | 2 +- scrapy/settings/default_settings.py | 10 +++++----- .../__init__.py | 0 .../depth.py | 0 .../httperror.py | 0 .../offsite.py | 0 .../referer.py | 0 .../urllength.py | 0 tests/test_spidermiddleware_depth.py | 4 ++-- tests/test_spidermiddleware_httperror.py | 2 +- tests/test_spidermiddleware_offsite.py | 2 +- tests/test_spidermiddleware_referer.py | 2 +- tests/test_spidermiddleware_urllength.py | 2 +- 16 files changed, 26 insertions(+), 26 deletions(-) rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/__init__.py (100%) rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/depth.py (100%) rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/httperror.py (100%) rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/offsite.py (100%) rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/referer.py (100%) rename scrapy/{contrib/spidermiddleware => spidermiddlewares}/urllength.py (100%) diff --git a/docs/faq.rst b/docs/faq.rst index 80d258b55..d98b06056 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,7 +149,7 @@ middleware (enabled by default) whose purpose is to filter out requests to domains outside the ones covered by the spider. For more info see: -:class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware`. +:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware`. What is the recommended way to deploy a Scrapy crawler in production? --------------------------------------------------------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3e68ad925..a049f1438 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -906,11 +906,11 @@ SPIDER_MIDDLEWARES_BASE Default:: { - 'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50, - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, - 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, - 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800, - 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900, + 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500, + 'scrapy.spidermiddlewares.referer.RefererMiddleware': 700, + 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800, + 'scrapy.spidermiddlewares.depth.DepthMiddleware': 900, } A dict containing the spider middlewares enabled by default in Scrapy. You @@ -1001,7 +1001,7 @@ URLLENGTH_LIMIT Default: ``2083`` -Scope: ``contrib.spidermiddleware.urllength`` +Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. For more information about the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index abeae2bce..6e82333f5 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -43,7 +43,7 @@ value. For example, if you want to disable the off-site middleware:: SPIDER_MIDDLEWARES = { 'myproject.middlewares.CustomSpiderMiddleware': 543, - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': None, + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -55,7 +55,7 @@ Writing your own spider middleware Each middleware component is a Python class that defines one or more of the following methods: -.. module:: scrapy.contrib.spidermiddleware +.. module:: scrapy.spidermiddlewares .. class:: SpiderMiddleware @@ -178,7 +178,7 @@ For a list of the components enabled by default (and their orders) see the DepthMiddleware --------------- -.. module:: scrapy.contrib.spidermiddleware.depth +.. module:: scrapy.spidermiddlewares.depth :synopsis: Depth Spider Middleware .. class:: DepthMiddleware @@ -199,7 +199,7 @@ DepthMiddleware HttpErrorMiddleware ------------------- -.. module:: scrapy.contrib.spidermiddleware.httperror +.. module:: scrapy.spidermiddlewares.httperror :synopsis: HTTP Error Spider Middleware .. class:: HttpErrorMiddleware @@ -264,7 +264,7 @@ Pass all responses, regardless of its status code. OffsiteMiddleware ----------------- -.. module:: scrapy.contrib.spidermiddleware.offsite +.. module:: scrapy.spidermiddlewares.offsite :synopsis: Offsite Spider Middleware .. class:: OffsiteMiddleware @@ -298,7 +298,7 @@ OffsiteMiddleware RefererMiddleware ----------------- -.. module:: scrapy.contrib.spidermiddleware.referer +.. module:: scrapy.spidermiddlewares.referer :synopsis: Referer Spider Middleware .. class:: RefererMiddleware @@ -323,7 +323,7 @@ Whether to enable referer middleware. UrlLengthMiddleware ------------------- -.. module:: scrapy.contrib.spidermiddleware.urllength +.. module:: scrapy.spidermiddlewares.urllength :synopsis: URL Length Spider Middleware .. class:: UrlLengthMiddleware diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index fdc5581a0..df030170c 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -77,7 +77,7 @@ scrapy.Spider An optional list of strings containing domains that this spider is allowed to crawl. Requests for URLs not belonging to the domain names specified in this list won't be followed if - :class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware` is enabled. + :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. .. attribute:: start_urls diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f4dcdc1a2..a7fe36648 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -224,11 +224,11 @@ SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side - 'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50, - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500, - 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700, - 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800, - 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900, + 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500, + 'scrapy.spidermiddlewares.referer.RefererMiddleware': 700, + 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800, + 'scrapy.spidermiddlewares.depth.DepthMiddleware': 900, # Spider side } diff --git a/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/spidermiddlewares/__init__.py similarity index 100% rename from scrapy/contrib/spidermiddleware/__init__.py rename to scrapy/spidermiddlewares/__init__.py diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/spidermiddlewares/depth.py similarity index 100% rename from scrapy/contrib/spidermiddleware/depth.py rename to scrapy/spidermiddlewares/depth.py diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/spidermiddlewares/httperror.py similarity index 100% rename from scrapy/contrib/spidermiddleware/httperror.py rename to scrapy/spidermiddlewares/httperror.py diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/spidermiddlewares/offsite.py similarity index 100% rename from scrapy/contrib/spidermiddleware/offsite.py rename to scrapy/spidermiddlewares/offsite.py diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/spidermiddlewares/referer.py similarity index 100% rename from scrapy/contrib/spidermiddleware/referer.py rename to scrapy/spidermiddlewares/referer.py diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/spidermiddlewares/urllength.py similarity index 100% rename from scrapy/contrib/spidermiddleware/urllength.py rename to scrapy/spidermiddlewares/urllength.py diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index e7ae75ed2..8120f1a95 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,6 +1,6 @@ from unittest import TestCase -from scrapy.contrib.spidermiddleware.depth import DepthMiddleware +from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.http import Response, Request from scrapy.spider import Spider from scrapy.statscol import StatsCollector @@ -37,7 +37,7 @@ class TestDepthMiddleware(TestCase): rdm = self.stats.get_value('request_depth_max', spider=self.spider) self.assertEquals(rdm, 1) - + def tearDown(self): self.stats.close_spider(self.spider, '') diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 503865f63..5cd2c2566 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -8,7 +8,7 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from scrapy.http import Response, Request from scrapy.spider import Spider -from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpError +from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index e5e99002a..296e8b1a3 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -4,7 +4,7 @@ from six.moves.urllib.parse import urlparse from scrapy.http import Response, Request from scrapy.spider import Spider -from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware +from scrapy.spidermiddlewares.offsite import OffsiteMiddleware from scrapy.utils.test import get_crawler class TestOffsiteMiddleware(TestCase): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index f408719d2..f2815ebd3 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.http import Response, Request from scrapy.spider import Spider -from scrapy.contrib.spidermiddleware.referer import RefererMiddleware +from scrapy.spidermiddlewares.referer import RefererMiddleware class TestRefererMiddleware(TestCase): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index ca2de18f2..1ef22ea07 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,6 +1,6 @@ from unittest import TestCase -from scrapy.contrib.spidermiddleware.urllength import UrlLengthMiddleware +from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spider import Spider From b2a15ddbf3ad39464670d847a26efc613a4f8a1f Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 21 Apr 2015 13:11:56 -0300 Subject: [PATCH 12/25] scrapy/contrib/spidermiddleware shims --- scrapy/contrib/spidermiddleware/__init__.py | 0 scrapy/contrib/spidermiddleware/depth.py | 7 +++++++ scrapy/contrib/spidermiddleware/httperror.py | 7 +++++++ scrapy/contrib/spidermiddleware/offsite.py | 7 +++++++ scrapy/contrib/spidermiddleware/referer.py | 7 +++++++ scrapy/contrib/spidermiddleware/urllength.py | 7 +++++++ 6 files changed, 35 insertions(+) create mode 100644 scrapy/contrib/spidermiddleware/__init__.py create mode 100644 scrapy/contrib/spidermiddleware/depth.py create mode 100644 scrapy/contrib/spidermiddleware/httperror.py create mode 100644 scrapy/contrib/spidermiddleware/offsite.py create mode 100644 scrapy/contrib/spidermiddleware/referer.py create mode 100644 scrapy/contrib/spidermiddleware/urllength.py diff --git a/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/contrib/spidermiddleware/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py new file mode 100644 index 000000000..718803148 --- /dev/null +++ b/scrapy/contrib/spidermiddleware/depth.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spidermiddleware.depth` is deprecated, " + "use `scrapy.spidermiddlewares.depth` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.spidermiddlewares.depth import * diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py new file mode 100644 index 000000000..e39fb3f56 --- /dev/null +++ b/scrapy/contrib/spidermiddleware/httperror.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spidermiddleware.httperror` is deprecated, " + "use `scrapy.spidermiddlewares.httperror` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.spidermiddlewares.httperror import * diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py new file mode 100644 index 000000000..a5ed9ea7e --- /dev/null +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spidermiddleware.offsite` is deprecated, " + "use `scrapy.spidermiddlewares.offsite` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.spidermiddlewares.offsite import * diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/contrib/spidermiddleware/referer.py new file mode 100644 index 000000000..fdf8d6659 --- /dev/null +++ b/scrapy/contrib/spidermiddleware/referer.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spidermiddleware.referer` is deprecated, " + "use `scrapy.spidermiddlewares.referer` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.spidermiddlewares.referer import * diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py new file mode 100644 index 000000000..5e51add59 --- /dev/null +++ b/scrapy/contrib/spidermiddleware/urllength.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spidermiddleware.urllength` is deprecated, " + "use `scrapy.spidermiddlewares.urllength` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.spidermiddlewares.urllength import * From fc346cba4d45fb5faf52983b8a25e2a3f5aa12f6 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 21 Apr 2015 13:20:08 -0300 Subject: [PATCH 13/25] Move scrapy/contrib/spiders to scrapy/spiders --- docs/intro/tutorial.rst | 2 +- docs/topics/commands.rst | 2 +- docs/topics/downloader-middleware.rst | 2 +- docs/topics/firebug.rst | 6 +++--- docs/topics/link-extractors.rst | 4 ++-- docs/topics/spiders.rst | 16 ++++++++-------- scrapy/contrib/spiders/__init__.py | 3 --- scrapy/spiders/__init__.py | 3 +++ scrapy/{contrib => }/spiders/crawl.py | 0 scrapy/{contrib => }/spiders/feed.py | 0 scrapy/{contrib => }/spiders/init.py | 0 scrapy/{contrib => }/spiders/sitemap.py | 0 scrapy/templates/spiders/crawl.tmpl | 2 +- scrapy/templates/spiders/csvfeed.tmpl | 2 +- scrapy/templates/spiders/xmlfeed.tmpl | 2 +- tests/test_spider.py | 4 ++-- tests/test_utils_spider.py | 2 +- 17 files changed, 25 insertions(+), 25 deletions(-) delete mode 100644 scrapy/contrib/spiders/__init__.py create mode 100644 scrapy/spiders/__init__.py rename scrapy/{contrib => }/spiders/crawl.py (100%) rename scrapy/{contrib => }/spiders/feed.py (100%) rename scrapy/{contrib => }/spiders/init.py (100%) rename scrapy/{contrib => }/spiders/sitemap.py (100%) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 479b17f99..219616587 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -511,7 +511,7 @@ using a `trick to pass additional data to the callbacks .. note:: As an example spider that leverages this mechanism, check out the - :class:`~scrapy.contrib.spiders.CrawlSpider` class for a generic spider + :class:`~scrapy.spiders.CrawlSpider` class for a generic spider that implements a small rules engine that you can use to write your crawlers on top of it. diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 638de5712..f1f79ce70 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -399,7 +399,7 @@ Supported options: * ``--pipelines``: process items through pipelines -* ``--rules`` or ``-r``: use :class:`~scrapy.contrib.spiders.CrawlSpider` +* ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` rules to discover the callback (i.e. spider method) to use for parsing the response diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index be19bc44f..bb969eca3 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -288,7 +288,7 @@ HttpAuthMiddleware Example:: - from scrapy.contrib.spiders import CrawlSpider + from scrapy.spiders import CrawlSpider class SomeIntranetSiteSpider(CrawlSpider): diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 3bbb902b2..359c99450 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -74,15 +74,15 @@ So, based on that regular expression we can create the first crawling rule:: follow=True, ), -The :class:`~scrapy.contrib.spiders.Rule` object instructs -:class:`~scrapy.contrib.spiders.CrawlSpider` based spiders how to follow the +The :class:`~scrapy.spiders.Rule` object instructs +:class:`~scrapy.spiders.CrawlSpider` based spiders how to follow the category links. ``parse_category`` will be a method of the spider which will process and extract data from those pages. This is how the spider would look so far:: from scrapy.linkextractors import LinkExtractor - from scrapy.contrib.spiders import CrawlSpider, Rule + from scrapy.spiders import CrawlSpider, Rule class GoogleDirectorySpider(CrawlSpider): name = 'directory.google.com' diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index be3eb4537..edb047c86 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -18,10 +18,10 @@ of :class:`scrapy.link.Link` objects. Link extractors are meant to be instantiated once and their ``extract_links`` method called several times with different responses to extract links to follow. -Link extractors are used in the :class:`~scrapy.contrib.spiders.CrawlSpider` +Link extractors are used in the :class:`~scrapy.spiders.CrawlSpider` class (available in Scrapy), through a set of rules, but you can also use it in your spiders, even if you don't subclass from -:class:`~scrapy.contrib.spiders.CrawlSpider`, as its purpose is very simple: to +:class:`~scrapy.spiders.CrawlSpider`, as its purpose is very simple: to extract links. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index df030170c..d2fdd61b8 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -319,7 +319,7 @@ with a ``TestItem`` declared in a ``myproject.items`` module:: description = scrapy.Field() -.. module:: scrapy.contrib.spiders +.. module:: scrapy.spiders :synopsis: Collection of generic spiders CrawlSpider @@ -394,7 +394,7 @@ CrawlSpider example Let's now take a look at an example CrawlSpider with rules:: import scrapy - from scrapy.contrib.spiders import CrawlSpider, Rule + from scrapy.spiders import CrawlSpider, Rule from scrapy.linkextractors import LinkExtractor class MySpider(CrawlSpider): @@ -515,7 +515,7 @@ XMLFeedSpider example These spiders are pretty easy to use, let's have a look at one example:: - from scrapy.contrib.spiders import XMLFeedSpider + from scrapy.spiders import XMLFeedSpider from myproject.items import TestItem class MySpider(XMLFeedSpider): @@ -575,7 +575,7 @@ CSVFeedSpider example Let's see an example similar to the previous one, but using a :class:`CSVFeedSpider`:: - from scrapy.contrib.spiders import CSVFeedSpider + from scrapy.spiders import CSVFeedSpider from myproject.items import TestItem class MySpider(CSVFeedSpider): @@ -669,7 +669,7 @@ SitemapSpider examples Simplest example: process all urls discovered through sitemaps using the ``parse`` callback:: - from scrapy.contrib.spiders import SitemapSpider + from scrapy.spiders import SitemapSpider class MySpider(SitemapSpider): sitemap_urls = ['http://www.example.com/sitemap.xml'] @@ -680,7 +680,7 @@ Simplest example: process all urls discovered through sitemaps using the Process some urls with certain callback and other urls with a different callback:: - from scrapy.contrib.spiders import SitemapSpider + from scrapy.spiders import SitemapSpider class MySpider(SitemapSpider): sitemap_urls = ['http://www.example.com/sitemap.xml'] @@ -698,7 +698,7 @@ callback:: Follow sitemaps defined in the `robots.txt`_ file and only follow sitemaps whose url contains ``/sitemap_shop``:: - from scrapy.contrib.spiders import SitemapSpider + from scrapy.spiders import SitemapSpider class MySpider(SitemapSpider): sitemap_urls = ['http://www.example.com/robots.txt'] @@ -712,7 +712,7 @@ whose url contains ``/sitemap_shop``:: Combine SitemapSpider with other sources of urls:: - from scrapy.contrib.spiders import SitemapSpider + from scrapy.spiders import SitemapSpider class MySpider(SitemapSpider): sitemap_urls = ['http://www.example.com/robots.txt'] diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py deleted file mode 100644 index c16bb6c0a..000000000 --- a/scrapy/contrib/spiders/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from scrapy.contrib.spiders.crawl import CrawlSpider, Rule -from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider -from scrapy.contrib.spiders.sitemap import SitemapSpider diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py new file mode 100644 index 000000000..de4f90d51 --- /dev/null +++ b/scrapy/spiders/__init__.py @@ -0,0 +1,3 @@ +from scrapy.spiders.crawl import CrawlSpider, Rule +from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider +from scrapy.spiders.sitemap import SitemapSpider diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/spiders/crawl.py similarity index 100% rename from scrapy/contrib/spiders/crawl.py rename to scrapy/spiders/crawl.py diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/spiders/feed.py similarity index 100% rename from scrapy/contrib/spiders/feed.py rename to scrapy/spiders/feed.py diff --git a/scrapy/contrib/spiders/init.py b/scrapy/spiders/init.py similarity index 100% rename from scrapy/contrib/spiders/init.py rename to scrapy/spiders/init.py diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/spiders/sitemap.py similarity index 100% rename from scrapy/contrib/spiders/sitemap.py rename to scrapy/spiders/sitemap.py diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index b84e785fd..a179d16ff 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import scrapy from scrapy.linkextractors import LinkExtractor -from scrapy.contrib.spiders import CrawlSpider, Rule +from scrapy.spiders import CrawlSpider, Rule from $project_name.items import ${ProjectName}Item diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 7e10cd8a8..69c606538 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from scrapy.contrib.spiders import CSVFeedSpider +from scrapy.spiders import CSVFeedSpider from $project_name.items import ${ProjectName}Item diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index a0e26aa14..9c0910d23 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from scrapy.contrib.spiders import XMLFeedSpider +from scrapy.spiders import XMLFeedSpider from $project_name.items import ${ProjectName}Item diff --git a/tests/test_spider.py b/tests/test_spider.py index 02352ec29..f771399ca 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -10,8 +10,8 @@ from scrapy import signals from scrapy.spider import Spider, BaseSpider from scrapy.settings import Settings from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse -from scrapy.contrib.spiders.init import InitSpider -from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \ +from scrapy.spiders.init import InitSpider +from scrapy.spiders import CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 334ec00f3..045e72117 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -3,7 +3,7 @@ from scrapy.http import Request from scrapy.item import BaseItem from scrapy.utils.spider import iterate_spider_output, iter_spider_classes -from scrapy.contrib.spiders import CrawlSpider +from scrapy.spiders import CrawlSpider class MyBaseSpider(CrawlSpider): From e262c5b8d5225897dda7976d2376c0ac29bf73ac Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 21 Apr 2015 13:26:53 -0300 Subject: [PATCH 14/25] scrapy/contrib/spiders shims --- scrapy/contrib/spiders/__init__.py | 7 +++++++ scrapy/contrib/spiders/crawl.py | 7 +++++++ scrapy/contrib/spiders/feed.py | 7 +++++++ scrapy/contrib/spiders/init.py | 7 +++++++ scrapy/contrib/spiders/sitemap.py | 7 +++++++ 5 files changed, 35 insertions(+) create mode 100644 scrapy/contrib/spiders/__init__.py create mode 100644 scrapy/contrib/spiders/crawl.py create mode 100644 scrapy/contrib/spiders/feed.py create mode 100644 scrapy/contrib/spiders/init.py create mode 100644 scrapy/contrib/spiders/sitemap.py diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py new file mode 100644 index 000000000..56780533b --- /dev/null +++ b/scrapy/contrib/spiders/__init__.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spiders` is deprecated, " + "use `scrapy.spiders` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.spiders import * diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py new file mode 100644 index 000000000..d20a8bb16 --- /dev/null +++ b/scrapy/contrib/spiders/crawl.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spiders.crawl` is deprecated, " + "use `scrapy.spiders.crawl` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.spiders.crawl import * diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py new file mode 100644 index 000000000..5eea9a062 --- /dev/null +++ b/scrapy/contrib/spiders/feed.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spiders.feed` is deprecated, " + "use `scrapy.spiders.feed` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.spiders.feed import * diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py new file mode 100644 index 000000000..6d1ec0aa9 --- /dev/null +++ b/scrapy/contrib/spiders/init.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spiders.init` is deprecated, " + "use `scrapy.spiders.init` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.spiders.init import * diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py new file mode 100644 index 000000000..2ad231fd8 --- /dev/null +++ b/scrapy/contrib/spiders/sitemap.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spiders.sitemap` is deprecated, " + "use `scrapy.spiders.sitemap` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.spiders.sitemap import * From 9a3e3ba505361efd12e49a42f1c77f5fa815b86a Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 21 Apr 2015 13:48:28 -0300 Subject: [PATCH 15/25] Move scrapy/contrib remaining top-level files to scrapy/extensions --- docs/topics/downloader-middleware.rst | 14 ++++----- docs/topics/extensions.rst | 30 +++++++++---------- docs/topics/feed-exports.rst | 12 ++++---- docs/topics/settings.rst | 28 ++++++++--------- scrapy/downloadermiddlewares/httpcache.py | 13 -------- scrapy/extensions/__init__.py | 0 scrapy/{contrib => extensions}/closespider.py | 0 scrapy/{contrib => extensions}/corestats.py | 0 scrapy/{contrib => extensions}/debug.py | 0 scrapy/{contrib => extensions}/feedexport.py | 0 scrapy/{contrib => extensions}/httpcache.py | 0 scrapy/{contrib => extensions}/logstats.py | 0 scrapy/{contrib => extensions}/memdebug.py | 0 scrapy/{contrib => extensions}/memusage.py | 0 scrapy/{contrib => extensions}/spiderstate.py | 0 scrapy/{contrib => extensions}/statsmailer.py | 0 scrapy/{contrib => extensions}/throttle.py | 0 scrapy/settings/default_settings.py | 30 +++++++++---------- .../templates/project/module/settings.py.tmpl | 2 +- tests/py3-ignores.txt | 2 ++ tests/test_downloadermiddleware_httpcache.py | 14 ++++----- ...ntrib_feedexport.py => test_feedexport.py} | 2 +- ...rib_spiderstate.py => test_spiderstate.py} | 2 +- 23 files changed, 69 insertions(+), 80 deletions(-) create mode 100644 scrapy/extensions/__init__.py rename scrapy/{contrib => extensions}/closespider.py (100%) rename scrapy/{contrib => extensions}/corestats.py (100%) rename scrapy/{contrib => extensions}/debug.py (100%) rename scrapy/{contrib => extensions}/feedexport.py (100%) rename scrapy/{contrib => extensions}/httpcache.py (100%) rename scrapy/{contrib => extensions}/logstats.py (100%) rename scrapy/{contrib => extensions}/memdebug.py (100%) rename scrapy/{contrib => extensions}/memusage.py (100%) rename scrapy/{contrib => extensions}/spiderstate.py (100%) rename scrapy/{contrib => extensions}/statsmailer.py (100%) rename scrapy/{contrib => extensions}/throttle.py (100%) rename tests/{test_contrib_feedexport.py => test_feedexport.py} (99%) rename tests/{test_contrib_spiderstate.py => test_spiderstate.py} (94%) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index bb969eca3..5cb6c9824 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -349,7 +349,7 @@ when an Internet connection is not available. The goal is to be able to In order to use this policy, set: -* :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.httpcache.DummyPolicy`` +* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.DummyPolicy`` .. _httpcache-policy-rfc2616: @@ -383,7 +383,7 @@ what is missing: In order to use this policy, set: -* :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.httpcache.RFC2616Policy`` +* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.RFC2616Policy`` .. _httpcache-storage-fs: @@ -395,7 +395,7 @@ File system storage backend is available for the HTTP cache middleware. In order to use this storage backend, set: -* :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.FilesystemCacheStorage`` +* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.FilesystemCacheStorage`` Each request/response pair is stored in a different directory containing the following files: @@ -430,7 +430,7 @@ By default, it uses the anydbm_ module, but you can change it with the In order to use this storage backend, set: -* :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.DbmCacheStorage`` +* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.DbmCacheStorage`` .. _httpcache-storage-leveldb: @@ -447,7 +447,7 @@ the scrapy shell in parallel for the same spider. In order to use this storage backend: -* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.LeveldbCacheStorage`` +* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.LeveldbCacheStorage`` * install `LevelDB python bindings`_ like ``pip install leveldb`` .. _LevelDB: http://code.google.com/p/leveldb/ @@ -536,7 +536,7 @@ Don't cache responses with these URI schemes. HTTPCACHE_STORAGE ^^^^^^^^^^^^^^^^^ -Default: ``'scrapy.contrib.httpcache.FilesystemCacheStorage'`` +Default: ``'scrapy.extensions.httpcache.FilesystemCacheStorage'`` The class which implements the cache storage backend. @@ -559,7 +559,7 @@ HTTPCACHE_POLICY .. versionadded:: 0.18 -Default: ``'scrapy.contrib.httpcache.DummyPolicy'`` +Default: ``'scrapy.extensions.httpcache.DummyPolicy'`` The class which implements the cache policy. diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 19c296651..d5d985087 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -35,7 +35,7 @@ your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented by a string: the full Python path to the extension's class name. For example:: EXTENSIONS = { - 'scrapy.contrib.corestats.CoreStats': 500, + 'scrapy.extensions.corestats.CoreStats': 500, 'scrapy.telnet.TelnetConsole': 500, } @@ -69,7 +69,7 @@ included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to ``None``. For example:: EXTENSIONS = { - 'scrapy.contrib.corestats.CoreStats': None, + 'scrapy.extensions.corestats.CoreStats': None, } Writing your own extension @@ -158,7 +158,7 @@ General purpose extensions Log Stats extension ~~~~~~~~~~~~~~~~~~~ -.. module:: scrapy.contrib.logstats +.. module:: scrapy.extensions.logstats :synopsis: Basic stats logging .. class:: LogStats @@ -168,7 +168,7 @@ Log basic stats like crawled pages and scraped items. Core Stats extension ~~~~~~~~~~~~~~~~~~~~ -.. module:: scrapy.contrib.corestats +.. module:: scrapy.extensions.corestats :synopsis: Core stats collection .. class:: CoreStats @@ -198,10 +198,10 @@ setting, and the server will listen in the port specified in Memory usage extension ~~~~~~~~~~~~~~~~~~~~~~ -.. module:: scrapy.contrib.memusage +.. module:: scrapy.extensions.memusage :synopsis: Memory usage extension -.. class:: scrapy.contrib.memusage.MemoryUsage +.. class:: scrapy.extensions.memusage.MemoryUsage .. note:: This extension does not work in Windows. @@ -226,10 +226,10 @@ can be configured with the following settings: Memory debugger extension ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. module:: scrapy.contrib.memdebug +.. module:: scrapy.extensions.memdebug :synopsis: Memory debugger extension -.. class:: scrapy.contrib.memdebug.MemoryDebugger +.. class:: scrapy.extensions.memdebug.MemoryDebugger An extension for debugging memory usage. It collects information about: @@ -242,10 +242,10 @@ info will be stored in the stats. Close spider extension ~~~~~~~~~~~~~~~~~~~~~~ -.. module:: scrapy.contrib.closespider +.. module:: scrapy.extensions.closespider :synopsis: Close spider extension -.. class:: scrapy.contrib.closespider.CloseSpider +.. class:: scrapy.extensions.closespider.CloseSpider Closes a spider automatically when some conditions are met, using a specific closing reason for each condition. @@ -313,17 +313,17 @@ set), spiders won't be closed by number of errors. StatsMailer extension ~~~~~~~~~~~~~~~~~~~~~ -.. module:: scrapy.contrib.statsmailer +.. module:: scrapy.extensions.statsmailer :synopsis: StatsMailer extension -.. class:: scrapy.contrib.statsmailer.StatsMailer +.. class:: scrapy.extensions.statsmailer.StatsMailer This simple extension can be used to send a notification e-mail every time a domain has finished scraping, including the Scrapy stats collected. The email will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS` setting. -.. module:: scrapy.contrib.debug +.. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy Debugging extensions @@ -332,7 +332,7 @@ Debugging extensions Stack trace dump extension ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. class:: scrapy.contrib.debug.StackTraceDump +.. class:: scrapy.extensions.debug.StackTraceDump Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_ signal is received. The information dumped is the following: @@ -361,7 +361,7 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal: Debugger extension ~~~~~~~~~~~~~~~~~~ -.. class:: scrapy.contrib.debug.Debugger +.. class:: scrapy.extensions.debug.Debugger Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_ signal is received. After the debugger is exited, the Scrapy process continues diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 27d601a19..faf9abc1c 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -209,7 +209,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORE_EMPTY` * :setting:`FEED_EXPORT_FIELDS` -.. currentmodule:: scrapy.contrib.feedexport +.. currentmodule:: scrapy.extensions.feedexport .. setting:: FEED_URI @@ -272,11 +272,11 @@ FEED_STORAGES_BASE Default:: { - '': 'scrapy.contrib.feedexport.FileFeedStorage', - 'file': 'scrapy.contrib.feedexport.FileFeedStorage', - 'stdout': 'scrapy.contrib.feedexport.StdoutFeedStorage', - 's3': 'scrapy.contrib.feedexport.S3FeedStorage', - 'ftp': 'scrapy.contrib.feedexport.FTPFeedStorage', + '': 'scrapy.extensions.feedexport.FileFeedStorage', + 'file': 'scrapy.extensions.feedexport.FileFeedStorage', + 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', + 's3': 'scrapy.extensions.feedexport.S3FeedStorage', + 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } A dict containing the built-in feed storage backends supported by Scrapy. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a049f1438..f331b1dba 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -536,15 +536,15 @@ EXTENSIONS_BASE Default:: { - 'scrapy.contrib.corestats.CoreStats': 0, + 'scrapy.extensions.corestats.CoreStats': 0, 'scrapy.telnet.TelnetConsole': 0, - 'scrapy.contrib.memusage.MemoryUsage': 0, - 'scrapy.contrib.memdebug.MemoryDebugger': 0, - 'scrapy.contrib.closespider.CloseSpider': 0, - 'scrapy.contrib.feedexport.FeedExporter': 0, - 'scrapy.contrib.logstats.LogStats': 0, - 'scrapy.contrib.spiderstate.SpiderState': 0, - 'scrapy.contrib.throttle.AutoThrottle': 0, + 'scrapy.extensions.memusage.MemoryUsage': 0, + 'scrapy.extensions.memdebug.MemoryDebugger': 0, + 'scrapy.extensions.closespider.CloseSpider': 0, + 'scrapy.extensions.feedexport.FeedExporter': 0, + 'scrapy.extensions.logstats.LogStats': 0, + 'scrapy.extensions.spiderstate.SpiderState': 0, + 'scrapy.extensions.throttle.AutoThrottle': 0, } The list of available extensions. Keep in mind that some of them need to @@ -689,7 +689,7 @@ MEMUSAGE_ENABLED Default: ``False`` -Scope: ``scrapy.contrib.memusage`` +Scope: ``scrapy.extensions.memusage`` Whether to enable the memory usage extension that will shutdown the Scrapy process when it exceeds a memory limit, and also notify by email when that @@ -704,7 +704,7 @@ MEMUSAGE_LIMIT_MB Default: ``0`` -Scope: ``scrapy.contrib.memusage`` +Scope: ``scrapy.extensions.memusage`` The maximum amount of memory to allow (in megabytes) before shutting down Scrapy (if MEMUSAGE_ENABLED is True). If zero, no check will be performed. @@ -718,7 +718,7 @@ MEMUSAGE_NOTIFY_MAIL Default: ``False`` -Scope: ``scrapy.contrib.memusage`` +Scope: ``scrapy.extensions.memusage`` A list of emails to notify if the memory limit has been reached. @@ -735,7 +735,7 @@ MEMUSAGE_REPORT Default: ``False`` -Scope: ``scrapy.contrib.memusage`` +Scope: ``scrapy.extensions.memusage`` Whether to send a memory usage report after each spider has been closed. @@ -748,7 +748,7 @@ MEMUSAGE_WARNING_MB Default: ``0`` -Scope: ``scrapy.contrib.memusage`` +Scope: ``scrapy.extensions.memusage`` The maximum amount of memory to allow (in megabytes) before sending a warning email notifying about it. If zero, no warning will be produced. @@ -961,7 +961,7 @@ STATSMAILER_RCPTS Default: ``[]`` (empty list) Send Scrapy stats after spiders finish scraping. See -:class:`~scrapy.contrib.statsmailer.StatsMailer` for more info. +:class:`~scrapy.extensions.statsmailer.StatsMailer` for more info. .. setting:: TELNETCONSOLE_ENABLED diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 7b4b53f7c..bd112c48d 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -90,16 +90,3 @@ class HttpCacheMiddleware(object): self.storage.store_response(spider, request, response) else: self.stats.inc_value('httpcache/uncacheable', spider=spider) - - -from scrapy.contrib.httpcache import FilesystemCacheStorage as _FilesystemCacheStorage -class FilesystemCacheStorage(_FilesystemCacheStorage): - - def __init__(self, *args, **kwargs): - import warnings - from scrapy.exceptions import ScrapyDeprecationWarning - warnings.warn('Importing FilesystemCacheStorage from ' - 'scrapy.contrib.downloadermiddlware.httpcache is ' - 'deprecated, use scrapy.contrib.httpcache instead.', - category=ScrapyDeprecationWarning, stacklevel=1) - super(FilesystemCacheStorage, self).__init__(*args, **kwargs) diff --git a/scrapy/extensions/__init__.py b/scrapy/extensions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/contrib/closespider.py b/scrapy/extensions/closespider.py similarity index 100% rename from scrapy/contrib/closespider.py rename to scrapy/extensions/closespider.py diff --git a/scrapy/contrib/corestats.py b/scrapy/extensions/corestats.py similarity index 100% rename from scrapy/contrib/corestats.py rename to scrapy/extensions/corestats.py diff --git a/scrapy/contrib/debug.py b/scrapy/extensions/debug.py similarity index 100% rename from scrapy/contrib/debug.py rename to scrapy/extensions/debug.py diff --git a/scrapy/contrib/feedexport.py b/scrapy/extensions/feedexport.py similarity index 100% rename from scrapy/contrib/feedexport.py rename to scrapy/extensions/feedexport.py diff --git a/scrapy/contrib/httpcache.py b/scrapy/extensions/httpcache.py similarity index 100% rename from scrapy/contrib/httpcache.py rename to scrapy/extensions/httpcache.py diff --git a/scrapy/contrib/logstats.py b/scrapy/extensions/logstats.py similarity index 100% rename from scrapy/contrib/logstats.py rename to scrapy/extensions/logstats.py diff --git a/scrapy/contrib/memdebug.py b/scrapy/extensions/memdebug.py similarity index 100% rename from scrapy/contrib/memdebug.py rename to scrapy/extensions/memdebug.py diff --git a/scrapy/contrib/memusage.py b/scrapy/extensions/memusage.py similarity index 100% rename from scrapy/contrib/memusage.py rename to scrapy/extensions/memusage.py diff --git a/scrapy/contrib/spiderstate.py b/scrapy/extensions/spiderstate.py similarity index 100% rename from scrapy/contrib/spiderstate.py rename to scrapy/extensions/spiderstate.py diff --git a/scrapy/contrib/statsmailer.py b/scrapy/extensions/statsmailer.py similarity index 100% rename from scrapy/contrib/statsmailer.py rename to scrapy/extensions/statsmailer.py diff --git a/scrapy/contrib/throttle.py b/scrapy/extensions/throttle.py similarity index 100% rename from scrapy/contrib/throttle.py rename to scrapy/extensions/throttle.py diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a7fe36648..9debaabc3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -113,15 +113,15 @@ except KeyError: EXTENSIONS = {} EXTENSIONS_BASE = { - 'scrapy.contrib.corestats.CoreStats': 0, + 'scrapy.extensions.corestats.CoreStats': 0, 'scrapy.telnet.TelnetConsole': 0, - 'scrapy.contrib.memusage.MemoryUsage': 0, - 'scrapy.contrib.memdebug.MemoryDebugger': 0, - 'scrapy.contrib.closespider.CloseSpider': 0, - 'scrapy.contrib.feedexport.FeedExporter': 0, - 'scrapy.contrib.logstats.LogStats': 0, - 'scrapy.contrib.spiderstate.SpiderState': 0, - 'scrapy.contrib.throttle.AutoThrottle': 0, + 'scrapy.extensions.memusage.MemoryUsage': 0, + 'scrapy.extensions.memdebug.MemoryDebugger': 0, + 'scrapy.extensions.closespider.CloseSpider': 0, + 'scrapy.extensions.feedexport.FeedExporter': 0, + 'scrapy.extensions.logstats.LogStats': 0, + 'scrapy.extensions.spiderstate.SpiderState': 0, + 'scrapy.extensions.throttle.AutoThrottle': 0, } FEED_URI = None @@ -131,11 +131,11 @@ FEED_STORE_EMPTY = False FEED_EXPORT_FIELDS = None FEED_STORAGES = {} FEED_STORAGES_BASE = { - '': 'scrapy.contrib.feedexport.FileFeedStorage', - 'file': 'scrapy.contrib.feedexport.FileFeedStorage', - 'stdout': 'scrapy.contrib.feedexport.StdoutFeedStorage', - 's3': 'scrapy.contrib.feedexport.S3FeedStorage', - 'ftp': 'scrapy.contrib.feedexport.FTPFeedStorage', + '': 'scrapy.extensions.feedexport.FileFeedStorage', + 'file': 'scrapy.extensions.feedexport.FileFeedStorage', + 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', + 's3': 'scrapy.extensions.feedexport.S3FeedStorage', + 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { @@ -151,12 +151,12 @@ FEED_EXPORTERS_BASE = { HTTPCACHE_ENABLED = False HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False -HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.FilesystemCacheStorage' +HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_DBM_MODULE = 'anydbm' -HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.DummyPolicy' +HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 37cbb4d32..11ff804ba 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -82,4 +82,4 @@ NEWSPIDER_MODULE = '$project_name.spiders' #HTTPCACHE_EXPIRATION_SECS=0 #HTTPCACHE_DIR='httpcache' #HTTPCACHE_IGNORE_HTTP_CODES=[] -#HTTPCACHE_STORAGE='scrapy.contrib.httpcache.FilesystemCacheStorage' +#HTTPCACHE_STORAGE='scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index a29b419b0..3d87bcb9a 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -100,7 +100,9 @@ scrapy/downloadermiddlewares/retry.py scrapy/downloadermiddlewares/httpproxy.py scrapy/downloadermiddlewares/cookies.py scrapy/contrib/statsmailer.py +scrapy/extensions/statsmailer.py scrapy/contrib/memusage.py +scrapy/extensions/memusage.py scrapy/commands/deploy.py scrapy/commands/bench.py scrapy/mail.py diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9c1678488..ac954cc15 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -17,8 +17,8 @@ from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware class _BaseTest(unittest.TestCase): - storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage' - policy_class = 'scrapy.contrib.httpcache.RFC2616Policy' + storage_class = 'scrapy.extensions.httpcache.DbmCacheStorage' + policy_class = 'scrapy.extensions.httpcache.RFC2616Policy' def setUp(self): self.yesterday = email.utils.formatdate(time.time() - 86400) @@ -127,7 +127,7 @@ class DefaultStorageTest(_BaseTest): class DbmStorageTest(DefaultStorageTest): - storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage' + storage_class = 'scrapy.extensions.httpcache.DbmCacheStorage' class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): @@ -146,7 +146,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): class FilesystemStorageTest(DefaultStorageTest): - storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage' + storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage' class FilesystemStorageGzipTest(FilesystemStorageTest): @@ -157,12 +157,12 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): class LeveldbStorageTest(DefaultStorageTest): pytest.importorskip('leveldb') - storage_class = 'scrapy.contrib.httpcache.LeveldbCacheStorage' + storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage' class DummyPolicyTest(_BaseTest): - policy_class = 'scrapy.contrib.httpcache.DummyPolicy' + policy_class = 'scrapy.extensions.httpcache.DummyPolicy' def test_middleware(self): with self._middleware() as mw: @@ -254,7 +254,7 @@ class DummyPolicyTest(_BaseTest): class RFC2616PolicyTest(DefaultStorageTest): - policy_class = 'scrapy.contrib.httpcache.RFC2616Policy' + policy_class = 'scrapy.extensions.httpcache.RFC2616Policy' def _process_requestresponse(self, mw, request, response): try: diff --git a/tests/test_contrib_feedexport.py b/tests/test_feedexport.py similarity index 99% rename from tests/test_contrib_feedexport.py rename to tests/test_feedexport.py index e6fd38ee3..41913e401 100644 --- a/tests/test_contrib_feedexport.py +++ b/tests/test_feedexport.py @@ -16,7 +16,7 @@ from tests.mockserver import MockServer from w3lib.url import path_to_file_uri import scrapy -from scrapy.contrib.feedexport import ( +from scrapy.extensions.feedexport import ( IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage ) diff --git a/tests/test_contrib_spiderstate.py b/tests/test_spiderstate.py similarity index 94% rename from tests/test_contrib_spiderstate.py rename to tests/test_spiderstate.py index 8e5897db7..1ddce4b99 100644 --- a/tests/test_contrib_spiderstate.py +++ b/tests/test_spiderstate.py @@ -2,7 +2,7 @@ import os from datetime import datetime from twisted.trial import unittest -from scrapy.contrib.spiderstate import SpiderState +from scrapy.extensions.spiderstate import SpiderState from scrapy.spider import Spider From fe4b260ef4579219aa53b5215f80da5a5daf5c3a Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 21 Apr 2015 13:54:28 -0300 Subject: [PATCH 16/25] Top-level scrapy/contrib shims --- scrapy/contrib/closespider.py | 7 +++++++ scrapy/contrib/corestats.py | 7 +++++++ scrapy/contrib/debug.py | 7 +++++++ scrapy/contrib/feedexport.py | 7 +++++++ scrapy/contrib/httpcache.py | 7 +++++++ scrapy/contrib/logstats.py | 7 +++++++ scrapy/contrib/memdebug.py | 7 +++++++ scrapy/contrib/memusage.py | 7 +++++++ scrapy/contrib/spiderstate.py | 7 +++++++ scrapy/contrib/statsmailer.py | 7 +++++++ scrapy/contrib/throttle.py | 7 +++++++ 11 files changed, 77 insertions(+) create mode 100644 scrapy/contrib/closespider.py create mode 100644 scrapy/contrib/corestats.py create mode 100644 scrapy/contrib/debug.py create mode 100644 scrapy/contrib/feedexport.py create mode 100644 scrapy/contrib/httpcache.py create mode 100644 scrapy/contrib/logstats.py create mode 100644 scrapy/contrib/memdebug.py create mode 100644 scrapy/contrib/memusage.py create mode 100644 scrapy/contrib/spiderstate.py create mode 100644 scrapy/contrib/statsmailer.py create mode 100644 scrapy/contrib/throttle.py diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py new file mode 100644 index 000000000..9c52c418f --- /dev/null +++ b/scrapy/contrib/closespider.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.closespider` is deprecated, " + "use `scrapy.extensions.closespider` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.closespider import * diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py new file mode 100644 index 000000000..2f5354239 --- /dev/null +++ b/scrapy/contrib/corestats.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.corestats` is deprecated, " + "use `scrapy.extensions.corestats` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.corestats import * diff --git a/scrapy/contrib/debug.py b/scrapy/contrib/debug.py new file mode 100644 index 000000000..a38f059ce --- /dev/null +++ b/scrapy/contrib/debug.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.debug` is deprecated, " + "use `scrapy.extensions.debug` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.debug import * diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py new file mode 100644 index 000000000..19651998a --- /dev/null +++ b/scrapy/contrib/feedexport.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.feedexport` is deprecated, " + "use `scrapy.extensions.feedexport` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.feedexport import * diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py new file mode 100644 index 000000000..196372fcb --- /dev/null +++ b/scrapy/contrib/httpcache.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.httpcache` is deprecated, " + "use `scrapy.extensions.httpcache` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.httpcache import * diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py new file mode 100644 index 000000000..62bc9b860 --- /dev/null +++ b/scrapy/contrib/logstats.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.logstats` is deprecated, " + "use `scrapy.extensions.logstats` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.logstats import * diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py new file mode 100644 index 000000000..4f6e4760e --- /dev/null +++ b/scrapy/contrib/memdebug.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.memdebug` is deprecated, " + "use `scrapy.extensions.memdebug` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.memdebug import * diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py new file mode 100644 index 000000000..e13bd78f3 --- /dev/null +++ b/scrapy/contrib/memusage.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.memusage` is deprecated, " + "use `scrapy.extensions.memusage` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.memusage import * diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py new file mode 100644 index 000000000..06afc8bfc --- /dev/null +++ b/scrapy/contrib/spiderstate.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.spiderstate` is deprecated, " + "use `scrapy.extensions.spiderstate` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.spiderstate import * diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py new file mode 100644 index 000000000..f9c9a37f5 --- /dev/null +++ b/scrapy/contrib/statsmailer.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.statsmailer` is deprecated, " + "use `scrapy.extensions.statsmailer` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.statsmailer import * diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py new file mode 100644 index 000000000..d5c234871 --- /dev/null +++ b/scrapy/contrib/throttle.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib.throttle` is deprecated, " + "use `scrapy.extensions.throttle` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.extensions.throttle import * From 645cdcbf9e6bbffd7fa7655091e64d67567c7f2f Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 23 Apr 2015 11:41:59 -0300 Subject: [PATCH 17/25] Rename scrapy/loader/processor.py to scrapy/loader/processors.py --- docs/topics/loaders.rst | 24 +++++++++---------- scrapy/contrib/loader/processor.py | 4 ++-- scrapy/loader/__init__.py | 2 +- scrapy/loader/{processor.py => processors.py} | 0 tests/test_loader.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) rename scrapy/loader/{processor.py => processors.py} (100%) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index a5efa3fb9..4c24166e6 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -151,7 +151,7 @@ Item Loaders are declared like Items, by using a class definition syntax. Here is an example:: from scrapy.loader import ItemLoader - from scrapy.loader.processor import TakeFirst, MapCompose, Join + from scrapy.loader.processors import TakeFirst, MapCompose, Join class ProductLoader(ItemLoader): @@ -182,7 +182,7 @@ output processors to use: in the :ref:`Item Field ` metadata. Here is an example:: import scrapy - from scrapy.loader.processor import Join, MapCompose, TakeFirst + from scrapy.loader.processors import Join, MapCompose, TakeFirst from w3lib.html import remove_tags def filter_price(value): @@ -309,7 +309,7 @@ ItemLoader objects Examples:: - >>> from scrapy.loader.processor import TakeFirst + >>> from scrapy.loader.processors import TakeFirst >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)') 'FOO` @@ -513,7 +513,7 @@ those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default Product Item Loader (``ProductLoader``):: - from scrapy.loader.processor import MapCompose + from scrapy.loader.processors import MapCompose from myproject.ItemLoaders import ProductLoader def strip_dashes(x): @@ -526,7 +526,7 @@ Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it:: - from scrapy.loader.processor import MapCompose + from scrapy.loader.processors import MapCompose from myproject.ItemLoaders import ProductLoader from myproject.utils.xml import remove_cdata @@ -551,7 +551,7 @@ needs. Available built-in processors ============================= -.. module:: scrapy.loader.processor +.. module:: scrapy.loader.processors :synopsis: A collection of processors to use with Item Loaders Even though you can use any callable function as input and output processors, @@ -570,7 +570,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.loader.processor import Identity + >>> from scrapy.loader.processors import Identity >>> proc = Identity() >>> proc(['one', 'two', 'three']) ['one', 'two', 'three'] @@ -583,7 +583,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.loader.processor import TakeFirst + >>> from scrapy.loader.processors import TakeFirst >>> proc = TakeFirst() >>> proc(['', 'one', 'two', 'three']) 'one' @@ -598,7 +598,7 @@ Here is a list of all built-in processors: Examples:: - >>> from scrapy.loader.processor import Join + >>> from scrapy.loader.processors import Join >>> proc = Join() >>> proc(['one', 'two', 'three']) u'one two three' @@ -619,7 +619,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.loader.processor import Compose + >>> from scrapy.loader.processors import Compose >>> proc = Compose(lambda v: v[0], str.upper) >>> proc(['hello', 'world']) 'HELLO' @@ -666,7 +666,7 @@ Here is a list of all built-in processors: >>> def filter_world(x): ... return None if x == 'world' else x ... - >>> from scrapy.loader.processor import MapCompose + >>> from scrapy.loader.processors import MapCompose >>> proc = MapCompose(filter_world, unicode.upper) >>> proc([u'hello', u'world', u'this', u'is', u'scrapy']) [u'HELLO, u'THIS', u'IS', u'SCRAPY'] @@ -683,7 +683,7 @@ Here is a list of all built-in processors: Example:: - >>> from scrapy.loader.processor import SelectJmes, Compose, MapCompose + >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries >>> proc({'foo': 'bar'}) 'bar' diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py index 68ddff4d4..da7e484a5 100644 --- a/scrapy/contrib/loader/processor.py +++ b/scrapy/contrib/loader/processor.py @@ -1,7 +1,7 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning warnings.warn("Module `scrapy.contrib.loader.processor` is deprecated, " - "use `scrapy.loader.processor` instead", + "use `scrapy.loader.processors` instead", ScrapyDeprecationWarning, stacklevel=2) -from scrapy.loader.processor import * +from scrapy.loader.processors import * diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index c861f8a24..b6d8506ca 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -14,7 +14,7 @@ from scrapy.utils.misc import arg_to_iter, extract_regex from scrapy.utils.python import flatten from .common import wrap_loader_context -from .processor import Identity +from .processors import Identity class ItemLoader(object): diff --git a/scrapy/loader/processor.py b/scrapy/loader/processors.py similarity index 100% rename from scrapy/loader/processor.py rename to scrapy/loader/processors.py diff --git a/tests/test_loader.py b/tests/test_loader.py index c51092ad0..6e8f7c0de 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -2,7 +2,7 @@ import unittest from functools import partial from scrapy.loader import ItemLoader -from scrapy.loader.processor import Join, Identity, TakeFirst, \ +from scrapy.loader.processors import Join, Identity, TakeFirst, \ Compose, MapCompose, SelectJmes from scrapy.item import Item, Field from scrapy.selector import Selector From 7a7c539116df18e43b6e46b510af0d28a19f5084 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 23 Apr 2015 11:44:49 -0300 Subject: [PATCH 18/25] Rename scrapy/utils/decorator.py to scrapy/utils/decorators.py --- scrapy/core/downloader/handlers/file.py | 2 +- scrapy/loader/__init__.py | 2 +- scrapy/selector/unified.py | 2 +- scrapy/utils/{decorator.py => decorators.py} | 0 scrapy/utils/response.py | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename scrapy/utils/{decorator.py => decorators.py} (100%) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 85bad1583..5a63e9d08 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -1,6 +1,6 @@ from w3lib.url import file_uri_to_path from scrapy.responsetypes import responsetypes -from scrapy.utils.decorator import defers +from scrapy.utils.decorators import defers class FileDownloadHandler(object): diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index b6d8506ca..84640f0b6 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -8,7 +8,7 @@ import six from scrapy.item import Item from scrapy.selector import Selector -from scrapy.utils.decorator import deprecated +from scrapy.utils.decorators import deprecated from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.misc import arg_to_iter, extract_regex from scrapy.utils.python import flatten diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 90dcfd686..164ded2bf 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -8,7 +8,7 @@ import six from scrapy.utils.misc import extract_regex from scrapy.utils.trackref import object_ref from scrapy.utils.python import unicode_to_str, flatten, iflatten -from scrapy.utils.decorator import deprecated +from scrapy.utils.decorators import deprecated from scrapy.http import HtmlResponse, XmlResponse from .lxmldocument import LxmlDocument from .csstranslator import ScrapyHTMLTranslator, ScrapyGenericTranslator diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorators.py similarity index 100% rename from scrapy/utils/decorator.py rename to scrapy/utils/decorators.py diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 1d79ec0e3..b5d7a58ca 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -13,7 +13,7 @@ from twisted.web import http from twisted.web.http import RESPONSES from w3lib import html -from scrapy.utils.decorator import deprecated +from scrapy.utils.decorators import deprecated @deprecated From 593b4ef56462b0de9224d05db6d91423e40acf85 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 23 Apr 2015 11:46:56 -0300 Subject: [PATCH 19/25] scrapy/utils/decorator.py shim --- scrapy/utils/decorator.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 scrapy/utils/decorator.py diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorator.py new file mode 100644 index 000000000..e8c8eae39 --- /dev/null +++ b/scrapy/utils/decorator.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.utils.decorator` is deprecated, " + "use `scrapy.utils.decorators` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.utils.decorators import * From f5bdf64f9cfd3802cc4d074878ce1d93f72a15db Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 23 Apr 2015 11:51:07 -0300 Subject: [PATCH 20/25] Rename scrapy/squeue.py to scrapy/squeues.py --- docs/faq.rst | 4 ++-- scrapy/settings/default_settings.py | 4 ++-- scrapy/{squeue.py => squeues.py} | 0 tests/{test_squeue.py => test_squeues.py} | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename scrapy/{squeue.py => squeues.py} (100%) rename tests/{test_squeue.py => test_squeues.py} (97%) diff --git a/docs/faq.rst b/docs/faq.rst index d98b06056..212baa585 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -94,8 +94,8 @@ in most cases. If you do want to crawl in true `BFO order`_, you can do it by setting the following settings:: DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleFifoDiskQueue' - SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.FifoMemoryQueue' + SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue' + SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue' My Scrapy crawler has memory leaks. What can I do? -------------------------------------------------- diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9debaabc3..f2098d626 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -215,8 +215,8 @@ RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False SCHEDULER = 'scrapy.core.scheduler.Scheduler' -SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleLifoDiskQueue' -SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.LifoMemoryQueue' +SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue' +SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue' SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' diff --git a/scrapy/squeue.py b/scrapy/squeues.py similarity index 100% rename from scrapy/squeue.py rename to scrapy/squeues.py diff --git a/tests/test_squeue.py b/tests/test_squeues.py similarity index 97% rename from tests/test_squeue.py rename to tests/test_squeues.py index 16014a897..b3e9a2208 100644 --- a/tests/test_squeue.py +++ b/tests/test_squeues.py @@ -1,5 +1,5 @@ from queuelib.tests import test_queue as t -from scrapy.squeue import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue +from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue from scrapy.item import Item, Field from scrapy.http import Request from scrapy.loader import ItemLoader From 78dcd4e1e71501da3e4e13780b6c6b89738d61c5 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 23 Apr 2015 11:52:58 -0300 Subject: [PATCH 21/25] scrapy/squeue.py shim --- scrapy/squeue.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 scrapy/squeue.py diff --git a/scrapy/squeue.py b/scrapy/squeue.py new file mode 100644 index 000000000..a4a3f4238 --- /dev/null +++ b/scrapy/squeue.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.squeue` is deprecated, " + "use `scrapy.squeues` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.squeues import * From 54a4ce069c884fba8edec1fe1b850ec5634e93f9 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 23 Apr 2015 11:57:14 -0300 Subject: [PATCH 22/25] Rename scrapy/dupefilter.py to scrapy/dupefilters.py --- docs/topics/settings.rst | 2 +- scrapy/{dupefilter.py => dupefilters.py} | 0 scrapy/settings/default_settings.py | 2 +- tests/py3-ignores.txt | 2 +- tests/{test_dupefilter.py => test_dupefilters.py} | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename scrapy/{dupefilter.py => dupefilters.py} (100%) rename tests/{test_dupefilter.py => test_dupefilters.py} (97%) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f331b1dba..f035bc33d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -487,7 +487,7 @@ If you want to disable it set to 0. DUPEFILTER_CLASS ---------------- -Default: ``'scrapy.dupefilter.RFPDupeFilter'`` +Default: ``'scrapy.dupefilters.RFPDupeFilter'`` The class used to detect and filter duplicate requests. diff --git a/scrapy/dupefilter.py b/scrapy/dupefilters.py similarity index 100% rename from scrapy/dupefilter.py rename to scrapy/dupefilters.py diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f2098d626..57dfc3256 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -100,7 +100,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { DOWNLOADER_STATS = True -DUPEFILTER_CLASS = 'scrapy.dupefilter.RFPDupeFilter' +DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter' try: EDITOR = os.environ['EDITOR'] diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 3d87bcb9a..e0871c350 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -25,7 +25,7 @@ tests/test_downloadermiddleware_retry.py tests/test_downloadermiddleware_robotstxt.py tests/test_downloadermiddleware_stats.py tests/test_downloadermiddleware_useragent.py -tests/test_dupefilter.py +tests/test_dupefilters.py tests/test_engine.py tests/test_http_cookies.py tests/test_http_request.py diff --git a/tests/test_dupefilter.py b/tests/test_dupefilters.py similarity index 97% rename from tests/test_dupefilter.py rename to tests/test_dupefilters.py index b0dd9546b..f8f800a72 100644 --- a/tests/test_dupefilter.py +++ b/tests/test_dupefilters.py @@ -1,7 +1,7 @@ import hashlib import unittest -from scrapy.dupefilter import RFPDupeFilter +from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request From 4ddf152be37db5e4307cb62f42389c5fc71b6814 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 23 Apr 2015 11:58:46 -0300 Subject: [PATCH 23/25] scrapy/dupefilter.py shim --- scrapy/dupefilter.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 scrapy/dupefilter.py diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py new file mode 100644 index 000000000..232d96288 --- /dev/null +++ b/scrapy/dupefilter.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.dupefilter` is deprecated, " + "use `scrapy.dupefilters` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.dupefilters import * From b827097c91eb27c5d747935155497cb87edcb816 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 23 Apr 2015 13:07:48 -0300 Subject: [PATCH 24/25] Rename scrapy/statscol.py to scrapy/statscollectors.py --- docs/topics/api.rst | 8 ++++---- docs/topics/settings.rst | 2 +- docs/topics/stats.rst | 2 +- scrapy/settings/default_settings.py | 2 +- scrapy/{statscol.py => statscollectors.py} | 0 tests/test_spidermiddleware_depth.py | 2 +- tests/test_stats.py | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) rename scrapy/{statscol.py => statscollectors.py} (100%) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 89e300b39..7805f63d3 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -65,7 +65,7 @@ how you :ref:`configure the downloader middlewares For an introduction on stats collection see :ref:`topics-stats`. - For the API see :class:`~scrapy.statscol.StatsCollector` class. + For the API see :class:`~scrapy.statscollectors.StatsCollector` class. .. attribute:: extensions @@ -452,11 +452,11 @@ Stats Collector API =================== There are several Stats Collectors available under the -:mod:`scrapy.statscol` module and they all implement the Stats -Collector API defined by the :class:`~scrapy.statscol.StatsCollector` +:mod:`scrapy.statscollectors` module and they all implement the Stats +Collector API defined by the :class:`~scrapy.statscollectors.StatsCollector` class (which they all inherit from). -.. module:: scrapy.statscol +.. module:: scrapy.statscollectors :synopsis: Stats Collectors .. class:: StatsCollector diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f035bc33d..26a6d762d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -936,7 +936,7 @@ Example:: STATS_CLASS ----------- -Default: ``'scrapy.statscol.MemoryStatsCollector'`` +Default: ``'scrapy.statscollectors.MemoryStatsCollector'`` The class to use for collecting stats, who must implement the :ref:`topics-api-stats`. diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 5560a9446..0837610d0 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -75,7 +75,7 @@ available in Scrapy which extend the basic Stats Collector. You can select which Stats Collector to use through the :setting:`STATS_CLASS` setting. The default Stats Collector used is the :class:`MemoryStatsCollector`. -.. module:: scrapy.statscol +.. module:: scrapy.statscollectors :synopsis: Stats Collectors MemoryStatsCollector diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 57dfc3256..7459c4d73 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,7 +234,7 @@ SPIDER_MIDDLEWARES_BASE = { SPIDER_MODULES = [] -STATS_CLASS = 'scrapy.statscol.MemoryStatsCollector' +STATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector' STATS_DUMP = True STATSMAILER_RCPTS = [] diff --git a/scrapy/statscol.py b/scrapy/statscollectors.py similarity index 100% rename from scrapy/statscol.py rename to scrapy/statscollectors.py diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 8120f1a95..5317795a3 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -3,7 +3,7 @@ from unittest import TestCase from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.http import Response, Request from scrapy.spider import Spider -from scrapy.statscol import StatsCollector +from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler diff --git a/tests/test_stats.py b/tests/test_stats.py index db1f50712..34f72736c 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,7 +1,7 @@ import unittest from scrapy.spider import Spider -from scrapy.statscol import StatsCollector, DummyStatsCollector +from scrapy.statscollectors import StatsCollector, DummyStatsCollector from scrapy.utils.test import get_crawler class StatsCollectorTest(unittest.TestCase): From 62191de6eae7bbad227361dded8d4726da0e8a11 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 23 Apr 2015 13:10:15 -0300 Subject: [PATCH 25/25] scrapy/statscol.py shim --- scrapy/statscol.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 scrapy/statscol.py diff --git a/scrapy/statscol.py b/scrapy/statscol.py new file mode 100644 index 000000000..b4ddcce28 --- /dev/null +++ b/scrapy/statscol.py @@ -0,0 +1,7 @@ +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.statscol` is deprecated, " + "use `scrapy.statscollectors` instead", + ScrapyDeprecationWarning, stacklevel=2) + +from scrapy.statscollectors import *