From a3a3107bc45483e0d7c77e45412121fdb8d539b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Nov 2019 09:46:54 +0100 Subject: [PATCH 1/8] MutableChain: return self from __iter__ --- scrapy/utils/python.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index ea5193f12..64402a2bb 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -388,9 +388,7 @@ class MutableChain(object): self.data = chain(self.data, *iterables) def __iter__(self): - return self.data.__iter__() + return self def __next__(self): return next(self.data) - - next = __next__ From 1a4a77d49fa580d35d1e023ab4b54a397b88088a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Nov 2019 10:24:31 +0100 Subject: [PATCH 2/8] Remove Python 2 check from MutableChainTest --- tests/test_utils_python.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3e1148354..6cae9793d 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -21,9 +21,8 @@ class MutableChainTest(unittest.TestCase): m.extend([7, 8]) m.extend([9, 10], (11, 12)) self.assertEqual(next(m), 0) - self.assertEqual(m.next(), 1) - self.assertEqual(m.__next__(), 2) - self.assertEqual(list(m), list(range(3, 13))) + self.assertEqual(m.__next__(), 1) + self.assertEqual(list(m), list(range(2, 13))) class ToUnicodeTest(unittest.TestCase): From 393a2a197251cd4ac10671fbbff11113be42d930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 18 Nov 2019 09:15:48 +0100 Subject: [PATCH 3/8] Include /requirements-py3.txt from /docs/requirements.txt --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index f9db85146..85812be9a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ +-r ../requirements-py3.txt Sphinx>=2.1 sphinx-notfound-page sphinx_rtd_theme From 99d8b05a0b1997033b2240aa9f945bbe659ee6dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 18 Nov 2019 10:58:47 +0100 Subject: [PATCH 4/8] Deprecate scrapy.utils.python.MutableChain.next --- scrapy/utils/python.py | 4 ++++ tests/test_utils_python.py | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 64402a2bb..6edf7d702 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -392,3 +392,7 @@ class MutableChain(object): def __next__(self): return next(self.data) + + @deprecated("scrapy.utils.python.MutableChain.__next__") + def next(self): + return self.__next__() diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 6cae9793d..ca2c241e4 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -5,6 +5,7 @@ import unittest from itertools import count import platform import six +from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, @@ -22,7 +23,12 @@ class MutableChainTest(unittest.TestCase): m.extend([9, 10], (11, 12)) self.assertEqual(next(m), 0) self.assertEqual(m.__next__(), 1) - self.assertEqual(list(m), list(range(2, 13))) + with catch_warnings(record=True) as warnings: + self.assertEqual(m.next(), 2) + self.assertEqual(len(warnings), 1) + self.assertIn('scrapy.utils.python.MutableChain.__next__', + str(warnings[0].message)) + self.assertEqual(list(m), list(range(3, 13))) class ToUnicodeTest(unittest.TestCase): From 74589df02f2961110166959b49002fd8e5037291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 18 Nov 2019 14:51:44 +0100 Subject: [PATCH 5/8] Make command doctests pass --- docs/topics/commands.rst | 28 ++++++++++++++++++++++++---- pytest.ini | 1 - 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index a93bee06b..5b3cd7e75 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -1,3 +1,5 @@ +.. highlight:: none + .. _topics-commands: ================= @@ -66,7 +68,9 @@ structure by default, similar to this:: The directory where the ``scrapy.cfg`` file resides is known as the *project root directory*. That file contains the name of the python module that defines -the project settings. Here is an example:: +the project settings. Here is an example: + +.. code-block:: ini [settings] default = myproject.settings @@ -80,7 +84,9 @@ A project root directory, the one that contains the ``scrapy.cfg``, may be shared by multiple Scrapy projects, each with its own settings module. In that case, you must define one or more aliases for those settings modules -under ``[settings]`` in your ``scrapy.cfg`` file:: +under ``[settings]`` in your ``scrapy.cfg`` file: + +.. code-block:: ini [settings] default = myproject1.settings @@ -277,6 +283,8 @@ check Run contract checks. +.. skip: start + Usage examples:: $ scrapy check -l @@ -294,6 +302,8 @@ Usage examples:: [FAILED] first_spider:parse >>> Returned 92 requests, expected 0..4 +.. skip: end + .. command:: list list @@ -481,6 +491,8 @@ Supported options: * ``--verbose`` or ``-v``: display information for each depth level +.. skip: start + Usage example:: $ scrapy parse http://www.example.com/ -c parse_item @@ -495,6 +507,8 @@ Usage example:: # Requests ----------------------------------------------------------------- [] +.. skip: end + .. command:: settings @@ -573,7 +587,9 @@ Default: ``''`` (empty string) A module to use for looking up custom Scrapy commands. This is used to add custom commands for your Scrapy project. -Example:: +Example: + +.. code-block:: python COMMANDS_MODULE = 'mybot.commands' @@ -588,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a ``scrapy.commands`` section in the entry points of the library ``setup.py`` file. -The following example adds ``my_command`` command:: +The following example adds ``my_command`` command: + +.. skip: next + +.. code-block:: python from setuptools import setup, find_packages diff --git a/pytest.ini b/pytest.ini index 529ad5d27..dab91416f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,7 +8,6 @@ addopts = --ignore=docs/_ext --ignore=docs/conf.py --ignore=docs/news.rst - --ignore=docs/topics/commands.rst --ignore=docs/topics/debug.rst --ignore=docs/topics/developer-tools.rst --ignore=docs/topics/dynamic-content.rst From e84cb18ca0b5b09c68cc76d6c48929d9ff933e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 18 Nov 2019 15:50:45 +0100 Subject: [PATCH 6/8] Use InterSphinx to link to the Twisted documentation --- docs/conf.py | 3 ++- docs/contributing.rst | 6 +++--- docs/topics/api.rst | 2 -- docs/topics/architecture.rst | 3 +-- docs/topics/email.rst | 13 +++++++------ docs/topics/item-pipeline.rst | 9 ++++----- docs/topics/media-pipeline.rst | 6 +++--- docs/topics/practices.rst | 3 +-- docs/topics/request-response.rst | 9 ++++----- docs/topics/signals.rst | 16 +++++++--------- scrapy/core/downloader/contextfactory.py | 17 ++++++++++------- scrapy/crawler.py | 15 ++++++++------- scrapy/signalmanager.py | 6 ++---- 13 files changed, 52 insertions(+), 56 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 6ec4582b1..6bfd2cb0e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -275,5 +275,6 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { 'python': ('https://docs.python.org/3', None), - 'sphinx': ('https://www.sphinx-doc.org/en/stable', None), + 'sphinx': ('https://www.sphinx-doc.org/en/master', None), + 'twisted': ('https://twistedmatrix.com/documents/current', None), } diff --git a/docs/contributing.rst b/docs/contributing.rst index f084bd23d..68ae2bf3c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -194,8 +194,9 @@ documentation instead of duplicating the docstring in files within the Tests ===== -Tests are implemented using the `Twisted unit-testing framework`_, running -tests requires `tox`_. +Tests are implemented using the :doc:`Twisted unit-testing framework +`. Running tests requires +`tox`_. .. _running-tests: @@ -269,7 +270,6 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users .. _Scrapy subreddit: https://reddit.com/r/scrapy -.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 7c8c40b5f..1c461a511 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -273,5 +273,3 @@ class (which they all inherit from). Close the given spider. After this is called, no more specific stats can be accessed or collected. - -.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 2effe94dc..ae25dfa2f 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -166,11 +166,10 @@ for concurrency. For more information about asynchronous programming and Twisted see these links: -* `Introduction to Deferreds in Twisted`_ +* :doc:`twisted:core/howto/defer-intro` * `Twisted - hello, asynchronous programming`_ * `Twisted Introduction - Krondo`_ .. _Twisted: https://twistedmatrix.com/trac/ -.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html .. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 12eedf2cd..72bf52227 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -9,13 +9,13 @@ Sending e-mail Although Python makes sending e-mails relatively easy via the `smtplib`_ library, Scrapy provides its own facility for sending e-mails which is very -easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid -interfering with the non-blocking IO of the crawler. It also provides a -simple API for sending attachments and it's very easy to configure, with a few -:ref:`settings `. +easy to use and it's implemented using :doc:`Twisted non-blocking IO +`, to avoid interfering with the non-blocking +IO of the crawler. It also provides a simple API for sending attachments and +it's very easy to configure, with a few :ref:`settings +`. .. _smtplib: https://docs.python.org/2/library/smtplib.html -.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html Quick example ============= @@ -39,7 +39,8 @@ MailSender class reference ========================== MailSender is the preferred class to use for sending emails from Scrapy, as it -uses `Twisted non-blocking IO`_, like the rest of the framework. +uses :doc:`Twisted non-blocking IO `, like the +rest of the framework. .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index fae18200a..cdc4953c2 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following This method is called for every item pipeline component. :meth:`process_item` must either: return a dict with data, return an :class:`~scrapy.item.Item` - (or any descendant class) object, return a `Twisted Deferred`_ or raise + (or any descendant class) object, return a + :class:`~twisted.internet.defer.Deferred` or raise :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer processed by further pipeline components. @@ -67,8 +68,6 @@ Additionally, they may also implement the following methods: :type crawler: :class:`~scrapy.crawler.Crawler` object -.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html - Item pipeline example ===================== @@ -166,7 +165,8 @@ method and how to clean up the resources properly.:: Take screenshot of item ----------------------- -This example demonstrates how to return Deferred_ from :meth:`process_item` method. +This example demonstrates how to return a +:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method. It uses Splash_ to render screenshot of item url. Pipeline makes request to locally running instance of Splash_. After request is downloaded and Deferred callback fires, it saves item to a file and adds filename to an item. @@ -209,7 +209,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite return item .. _Splash: https://splash.readthedocs.io/en/stable/ -.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html Duplicates filter ----------------- diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 431cc6027..206e7cfa5 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -441,8 +441,9 @@ See here the methods that you can override in your custom Files Pipeline: * ``success`` is a boolean which is ``True`` if the image was downloaded successfully or ``False`` if it failed for some reason - * ``file_info_or_error`` is a dict containing the following keys (if success - is ``True``) or a `Twisted Failure`_ if there was a problem. + * ``file_info_or_error`` is a dict containing the following keys (if + success is ``True``) or a :exc:`~twisted.python.failure.Failure` if + there was a problem. * ``url`` - the url where the file was downloaded from. This is the url of the request returned from the :meth:`~get_media_requests` @@ -577,5 +578,4 @@ above:: item['image_paths'] = image_paths return item -.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html .. _MD5 hash: https://en.wikipedia.org/wiki/MD5 diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index a6d4f0d6d..e3e8fdc72 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -101,7 +101,7 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -.. seealso:: `Twisted Reactor Overview`_. +.. seealso:: :doc:`twisted:core/howto/reactor-basics` .. _run-multiple-spiders: @@ -253,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders -.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html .. _Crawlera: https://scrapinghub.com/crawlera .. _scrapoxy: https://scrapoxy.io/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ee37f648e..4cf367d96 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -121,8 +121,8 @@ Request objects :param errback: a function that will be called if any exception was raised while processing the request. This includes pages that failed - with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance - as first parameter. + with 404 HTTP errors and such. It receives a + :exc:`~twisted.python.failure.Failure` as first parameter. For more information, see :ref:`topics-request-response-ref-errbacks` below. :type errback: callable @@ -254,8 +254,8 @@ Using errbacks to catch exceptions in request processing The errback of a request is a function that will be called when an exception is raise while processing it. -It receives a `Twisted Failure`_ instance as first parameter and can be -used to track connection establishment timeouts, DNS errors etc. +It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can +be used to track connection establishment timeouts, DNS errors etc. Here's an example spider logging all errors and catching some specific errors if needed:: @@ -816,5 +816,4 @@ XmlResponse objects adds encoding auto-discovering support by looking into the XML declaration line. See :attr:`TextResponse.encoding`. -.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html .. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241 diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ff07b9d55..3f29aa323 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -50,10 +50,10 @@ Here is a simple example showing how you can catch signals and perform some acti Deferred signal handlers ======================== -Some signals support returning `Twisted deferreds`_ from their handlers, see -the :ref:`topics-signals-ref` below to know which ones. +Some signals support returning :class:`~twisted.internet.defer.Deferred` +objects from their handlers, see the :ref:`topics-signals-ref` below to know +which ones. -.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html .. _topics-signals-ref: @@ -155,8 +155,8 @@ item_error :param spider: the spider which raised the exception :type spider: :class:`~scrapy.spiders.Spider` object - :param failure: the exception raised as a Twisted `Failure`_ object - :type failure: `Failure`_ object + :param failure: the exception raised + :type failure: twisted.python.failure.Failure spider_closed ------------- @@ -236,8 +236,8 @@ spider_error This signal does not support returning deferreds from their handlers. - :param failure: the exception raised as a Twisted `Failure`_ object - :type failure: `Failure`_ object + :param failure: the exception raised + :type failure: twisted.python.failure.Failure :param response: the response being processed when the exception was raised :type response: :class:`~scrapy.http.Response` object @@ -333,5 +333,3 @@ response_downloaded :param spider: the spider for which the response is intended :type spider: :class:`~scrapy.spiders.Spider` object - -.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 89d2776ae..6e023ebcc 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -67,15 +67,18 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): """ Twisted-recommended context factory for web clients. - Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: - "The default is to use a BrowserLikePolicyForHTTPS, - so unless you have special requirements you can leave this as-is." + Quoting the documentation of the :class:`~twisted.web.client.Agent` class: - creatorForNetloc() is the same as BrowserLikePolicyForHTTPS - except this context factory allows setting the TLS/SSL method to use. + The default is to use a + :class:`~twisted.web.client.BrowserLikePolicyForHTTPS`, so unless you + have special requirements you can leave this as-is. - Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) - which allows TLS protocol negotiation. + :meth:`creatorForNetloc` is the same as + :class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context + factory allows setting the TLS/SSL method to use. + + The default OpenSSL method is ``TLS_METHOD`` (also called + ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ def creatorForNetloc(self, hostname, port): diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 8868a985b..f8c80880a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -110,7 +110,7 @@ class Crawler(object): class CrawlerRunner(object): """ This is a convenient helper class that keeps track of, manages and runs - crawlers inside an already setup Twisted `reactor`_. + crawlers inside an already setup :mod:`~twisted.internet.reactor`. The CrawlerRunner object must be instantiated with a :class:`~scrapy.settings.Settings` object. @@ -233,12 +233,13 @@ class CrawlerProcess(CrawlerRunner): A class to run multiple scrapy crawlers in a process simultaneously. This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support - for starting a Twisted `reactor`_ and handling shutdown signals, like the - keyboard interrupt command Ctrl-C. It also configures top-level logging. + for starting a :mod:`~twisted.internet.reactor` and handling shutdown + signals, like the keyboard interrupt command Ctrl-C. It also configures + top-level logging. This utility should be a better fit than :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another - Twisted `reactor`_ within your application. + :mod:`~twisted.internet.reactor` within your application. The CrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. @@ -273,9 +274,9 @@ class CrawlerProcess(CrawlerRunner): def start(self, stop_after_crawl=True): """ - This method starts a Twisted `reactor`_, adjusts its pool size to - :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based - on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. + This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool + size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache + based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 296d27ed8..9a160f62e 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -46,16 +46,14 @@ class SignalManager(object): def send_catch_log_deferred(self, signal, **kwargs): """ - Like :meth:`send_catch_log` but supports returning `deferreds`_ from - signal handlers. + Like :meth:`send_catch_log` but supports returning + :class:`~twisted.internet.defer.Deferred` objects from signal handlers. Returns a Deferred that gets fired once all signal handlers deferreds were fired. Send a signal, catch exceptions and log them. The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). - - .. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html """ kwargs.setdefault('sender', self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) From fed93515de4e306eb3262125c09eb49decdb2944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 18 Nov 2019 16:11:03 +0100 Subject: [PATCH 7/8] Add tooltips to documentation cross-references --- docs/conf.py | 1 + docs/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 6ec4582b1..e2784cf17 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,6 +27,7 @@ sys.path.insert(0, path.dirname(path.dirname(__file__))) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ + 'hoverxref.extension', 'notfound.extension', 'scrapydocs', 'sphinx.ext.autodoc', diff --git a/docs/requirements.txt b/docs/requirements.txt index f9db85146..773b92cea 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ Sphinx>=2.1 +sphinx-hoverxref sphinx-notfound-page sphinx_rtd_theme From f261cf65e999573d95a575ba362c3e32b026f894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 18 Nov 2019 17:16:09 +0100 Subject: [PATCH 8/8] Add missing blank lines between functions and classes Also fixed 2 unrelated Flake8 issues --- pytest.ini | 118 ++++++++---------- scrapy/commands/fetch.py | 1 + scrapy/commands/list.py | 1 + scrapy/commands/settings.py | 1 + scrapy/commands/view.py | 1 + scrapy/core/downloader/handlers/datauri.py | 4 +- scrapy/downloadermiddlewares/ajaxcrawl.py | 2 + scrapy/dupefilters.py | 1 + scrapy/exceptions.py | 13 ++ scrapy/extension.py | 1 + scrapy/extensions/spiderstate.py | 1 + scrapy/http/response/html.py | 1 + scrapy/http/response/xml.py | 1 + scrapy/interfaces.py | 1 + scrapy/loader/common.py | 1 + scrapy/pipelines/__init__.py | 1 + scrapy/resolver.py | 1 + scrapy/robotstxt.py | 3 + scrapy/utils/console.py | 7 ++ scrapy/utils/decorators.py | 1 + scrapy/utils/defer.py | 9 ++ scrapy/utils/display.py | 3 + scrapy/utils/engine.py | 3 + scrapy/utils/ftp.py | 1 + scrapy/utils/gz.py | 1 + scrapy/utils/httpobj.py | 3 + scrapy/utils/job.py | 1 + scrapy/utils/python.py | 1 + scrapy/utils/reactor.py | 1 + scrapy/utils/request.py | 2 + scrapy/utils/response.py | 4 + scrapy/utils/spider.py | 1 + scrapy/utils/template.py | 2 + scrapy/utils/test.py | 6 + scrapy/utils/versions.py | 2 +- tests/mocks/dummydbm.py | 1 + tests/pipelines.py | 1 + tests/spiders.py | 1 + tests/test_cmdline/extensions.py | 1 + tests/test_command_parse.py | 1 + tests/test_dependencies.py | 1 + ...test_downloadermiddleware_ajaxcrawlable.py | 2 + tests/test_downloadermiddleware_httpcache.py | 1 + tests/test_dupefilters.py | 1 + tests/test_http_headers.py | 1 + tests/test_logformatter.py | 1 + tests/test_mail.py | 1 + tests/test_middleware.py | 4 + tests/test_responsetypes.py | 1 + tests/test_robotstxt_interface.py | 2 + tests/test_spiderloader/__init__.py | 1 + .../test_spiders/nested/spider4.py | 1 + .../test_spiderloader/test_spiders/spider0.py | 1 + .../test_spiderloader/test_spiders/spider1.py | 1 + .../test_spiderloader/test_spiders/spider2.py | 1 + .../test_spiderloader/test_spiders/spider3.py | 1 + tests/test_spidermiddleware_offsite.py | 2 + tests/test_spidermiddleware_output_chain.py | 4 + tests/test_spidermiddleware_referer.py | 4 + tests/test_squeues.py | 13 ++ tests/test_utils_console.py | 1 + tests/test_utils_defer.py | 9 ++ tests/test_utils_http.py | 1 + tests/test_utils_httpobj.py | 1 + tests/test_utils_iterators.py | 1 + tests/test_utils_request.py | 1 + tests/test_utils_signal.py | 1 + tests/test_utils_sitemap.py | 1 + tests/test_utils_spider.py | 3 + tests/test_utils_url.py | 2 + 70 files changed, 199 insertions(+), 72 deletions(-) diff --git a/pytest.ini b/pytest.ini index 529ad5d27..a3693a778 100644 --- a/pytest.ini +++ b/pytest.ini @@ -30,16 +30,15 @@ flake8-ignore = scrapy/commands/check.py F401 E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731 + scrapy/commands/fetch.py E401 E501 E128 E502 E731 scrapy/commands/genspider.py E128 E501 E502 - scrapy/commands/list.py E302 scrapy/commands/parse.py E128 E501 E731 E226 scrapy/commands/runspider.py E501 - scrapy/commands/settings.py E302 E128 + scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 E501 E502 scrapy/commands/startproject.py E502 E127 E501 E128 scrapy/commands/version.py E501 E128 - scrapy/commands/view.py F401 E302 + scrapy/commands/view.py F401 # scrapy/contracts scrapy/contracts/__init__.py E501 W504 scrapy/contracts/default.py E502 E128 @@ -60,7 +59,7 @@ flake8-ignore = scrapy/core/downloader/handlers/http11.py E501 scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126 # scrapy/downloadermiddlewares - scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226 + scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226 scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 scrapy/downloadermiddlewares/httpcache.py E501 E126 @@ -72,11 +71,11 @@ flake8-ignore = scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions scrapy/extensions/closespider.py E501 E502 E128 E123 - scrapy/extensions/corestats.py E302 E501 + scrapy/extensions/corestats.py E501 scrapy/extensions/feedexport.py E128 E501 scrapy/extensions/httpcache.py E128 E501 E303 F401 scrapy/extensions/memdebug.py E501 - scrapy/extensions/spiderstate.py E302 E501 + scrapy/extensions/spiderstate.py E501 scrapy/extensions/telnet.py E501 W504 scrapy/extensions/throttle.py E501 # scrapy/http @@ -87,18 +86,14 @@ flake8-ignore = scrapy/http/request/form.py E501 E123 scrapy/http/request/json_request.py E501 scrapy/http/response/__init__.py E501 E128 W293 W291 - scrapy/http/response/html.py E302 scrapy/http/response/text.py E501 W293 E128 E124 - scrapy/http/response/xml.py E302 # scrapy/linkextractors scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401 scrapy/linkextractors/lxmlhtml.py E501 E731 E226 # scrapy/loader scrapy/loader/__init__.py E501 E502 E128 - scrapy/loader/common.py E302 scrapy/loader/processors.py E501 # scrapy/pipelines - scrapy/pipelines/__init__.py E302 scrapy/pipelines/files.py E116 E501 E266 scrapy/pipelines/images.py E265 E501 scrapy/pipelines/media.py E125 E501 E266 @@ -123,56 +118,50 @@ flake8-ignore = scrapy/utils/benchserver.py E501 scrapy/utils/boto.py F401 scrapy/utils/conf.py E402 E502 E501 - scrapy/utils/console.py E302 E261 F401 E306 E305 + scrapy/utils/console.py E261 F401 E306 E305 scrapy/utils/curl.py F401 scrapy/utils/datatypes.py E501 E226 - scrapy/utils/decorators.py E501 E302 - scrapy/utils/defer.py E501 E302 E128 + scrapy/utils/decorators.py E501 + scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 E502 - scrapy/utils/display.py E302 - scrapy/utils/engine.py F401 E261 E302 - scrapy/utils/ftp.py E302 - scrapy/utils/gz.py E305 E501 E302 W504 + scrapy/utils/engine.py F401 E261 + scrapy/utils/gz.py E305 E501 W504 scrapy/utils/http.py F403 F401 E226 - scrapy/utils/httpobj.py E302 E501 + scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 E701 - scrapy/utils/job.py E302 scrapy/utils/log.py E128 W503 scrapy/utils/markup.py F403 F401 W292 scrapy/utils/misc.py E501 E226 scrapy/utils/multipart.py F403 F401 W292 scrapy/utils/project.py E501 - scrapy/utils/python.py E501 E302 - scrapy/utils/reactor.py E302 E226 + scrapy/utils/python.py E501 + scrapy/utils/reactor.py E226 scrapy/utils/reqser.py E501 - scrapy/utils/request.py E302 E127 E501 - scrapy/utils/response.py E501 E302 E128 + scrapy/utils/request.py E127 E501 + scrapy/utils/response.py E501 E128 scrapy/utils/signal.py E501 E128 scrapy/utils/sitemap.py E501 - scrapy/utils/spider.py E271 E302 E501 + scrapy/utils/spider.py E271 E501 scrapy/utils/ssl.py E501 - scrapy/utils/template.py E302 - scrapy/utils/test.py E302 E501 + scrapy/utils/test.py E501 scrapy/utils/url.py E501 F403 F401 E128 F405 # scrapy scrapy/__init__.py E402 E501 scrapy/_monkeypatches.py W293 scrapy/cmdline.py E502 E501 scrapy/crawler.py E501 - scrapy/dupefilters.py E302 E501 E202 - scrapy/exceptions.py E302 E501 + scrapy/dupefilters.py E501 E202 + scrapy/exceptions.py E501 scrapy/exporters.py E501 E261 E226 - scrapy/extension.py E302 - scrapy/interfaces.py E302 E501 + scrapy/interfaces.py E501 scrapy/item.py E501 E128 scrapy/link.py E501 scrapy/logformatter.py E501 W293 scrapy/mail.py E402 E128 E501 E502 scrapy/middleware.py E502 E128 E501 scrapy/pqueues.py E501 - scrapy/resolver.py E302 scrapy/responsetypes.py E128 E501 E305 - scrapy/robotstxt.py E302 E501 + scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 scrapy/spiderloader.py E225 F841 E501 E126 @@ -181,91 +170,82 @@ flake8-ignore = # tests tests/__init__.py F401 E402 E501 tests/mockserver.py E401 E501 E126 E123 F401 - tests/pipelines.py E302 F841 E226 - tests/spiders.py E302 E501 E127 + tests/pipelines.py F841 E226 + tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 tests/test_command_fetch.py E501 E261 - tests/test_command_parse.py F401 E302 E501 E128 E303 E226 + tests/test_command_parse.py F401 E501 E128 E303 E226 tests/test_command_shell.py E501 E128 tests/test_commands.py F401 E128 E501 tests/test_contracts.py E501 E128 W293 tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E306 E501 - tests/test_dependencies.py E302 F841 E501 E305 + tests/test_dependencies.py F841 E501 E305 tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123 tests/test_downloadermiddleware.py E501 - tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501 + tests/test_downloadermiddleware_ajaxcrawlable.py E501 tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126 tests/test_downloadermiddleware_decompression.py E127 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 - tests/test_downloadermiddleware_httpcache.py E501 E302 E305 F401 + tests/test_downloadermiddleware_httpcache.py E501 E305 F401 tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123 tests/test_downloadermiddleware_httpproxy.py F401 E501 E128 tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305 tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124 + tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124 tests/test_engine.py E401 E501 E502 E128 E261 tests/test_exporters.py E501 E731 E306 E128 E124 tests/test_extension_telnet.py F401 F841 tests/test_feedexport.py E501 F401 F841 E241 tests/test_http_cookies.py E501 - tests/test_http_headers.py E302 E501 + tests/test_http_headers.py E501 tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123 tests/test_http_response.py E501 E301 E502 E128 E265 tests/test_item.py E701 E128 F841 E306 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 E124 - tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241 - tests/test_logformatter.py E128 E501 E122 E302 - tests/test_mail.py E302 E128 E501 E305 - tests/test_middleware.py E302 E501 E128 + tests/test_loader.py E501 E731 E303 E741 E128 E117 E241 + tests/test_logformatter.py E128 E501 E122 + tests/test_mail.py E128 E501 E305 + tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E131 E501 E128 E126 tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226 tests/test_pipeline_images.py F401 F841 E501 E303 tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502 tests/test_request_cb_kwargs.py E501 - tests/test_responsetypes.py E501 E302 E305 - tests/test_robotstxt_interface.py F401 E302 E501 W291 E501 + tests/test_responsetypes.py E501 E305 + tests/test_robotstxt_interface.py F401 E501 W291 E501 tests/test_scheduler.py E501 E126 E123 tests/test_selector.py F401 E501 E127 tests/test_spider.py E501 F401 tests/test_spidermiddleware.py E501 E226 tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 - tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293 - tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226 - tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121 - tests/test_squeues.py E501 E302 E701 E741 + tests/test_spidermiddleware_offsite.py E501 E128 E111 W293 + tests/test_spidermiddleware_output_chain.py F401 E501 W293 E226 + tests/test_spidermiddleware_referer.py F401 E501 F841 E125 E201 E261 E124 E501 E241 E121 + tests/test_squeues.py E501 E701 E741 tests/test_utils_conf.py E501 E303 E128 - tests/test_utils_console.py E302 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 E305 - tests/test_utils_defer.py E306 E261 E501 E302 F841 E226 + tests/test_utils_defer.py E306 E261 E501 F841 E226 tests/test_utils_deprecate.py F841 E306 E501 - tests/test_utils_http.py E302 E501 E502 E128 W504 - tests/test_utils_httpobj.py E302 - tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241 + tests/test_utils_http.py E501 E502 E128 W504 + tests/test_utils_iterators.py E501 E128 E129 E303 E241 tests/test_utils_log.py E741 E226 tests/test_utils_python.py E501 E303 E731 E701 E305 tests/test_utils_reqser.py F401 E501 E128 - tests/test_utils_request.py E302 E501 E128 E305 + tests/test_utils_request.py E501 E128 E305 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 F841 E302 E731 E226 - tests/test_utils_sitemap.py E302 E128 E501 E124 - tests/test_utils_spider.py E261 E302 E305 + tests/test_utils_signal.py E741 F841 E731 E226 + tests/test_utils_sitemap.py E128 E501 E124 + tests/test_utils_spider.py E261 E305 tests/test_utils_template.py E305 - tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123 + tests/test_utils_url.py F401 E501 E127 E305 E211 E125 E501 E226 E241 E126 E123 tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126 - tests/mocks/dummydbm.py E302 tests/test_cmdline/__init__.py E502 E501 - tests/test_cmdline/extensions.py E302 tests/test_settings/__init__.py F401 E501 E128 - tests/test_spiderloader/__init__.py E128 E501 E302 - tests/test_spiderloader/test_spiders/spider0.py E302 - tests/test_spiderloader/test_spiders/spider1.py E302 - tests/test_spiderloader/test_spiders/spider2.py E302 - tests/test_spiderloader/test_spiders/spider3.py E302 - tests/test_spiderloader/test_spiders/nested/spider4.py E302 + tests/test_spiderloader/__init__.py E128 E501 tests/test_utils_misc/__init__.py E501 diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index d45133e0e..724b4a1c4 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -8,6 +8,7 @@ from scrapy.exceptions import UsageError from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import spidercls_for_request, DefaultSpider + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index a255b3b94..422183ac1 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,6 +1,7 @@ from __future__ import print_function from scrapy.commands import ScrapyCommand + class Command(ScrapyCommand): requires_project = True diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index bee52f06a..ffe3aa2eb 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -4,6 +4,7 @@ import json from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 59e665016..31c17c0ab 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,6 +1,7 @@ from scrapy.commands import fetch, ScrapyCommand from scrapy.utils.response import open_in_browser + class Command(fetch.Command): def short_desc(self): diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index ad25beb3b..9e5020753 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -17,8 +17,8 @@ class DataURIDownloadHandler(object): respcls = responsetypes.from_mimetype(uri.media_type) resp_kwargs = {} - if (issubclass(respcls, TextResponse) and - uri.media_type.split('/')[0] == 'text'): + if (issubclass(respcls, TextResponse) + and uri.media_type.split('/')[0] == 'text'): charset = uri.media_type_parameters.get('charset') resp_kwargs['encoding'] = charset diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 72715dba7..ba50793bb 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -68,6 +68,8 @@ class AjaxCrawlMiddleware(object): # XXX: move it to w3lib? _ajax_crawlable_re = re.compile(six.u(r'')) + + def _has_ajaxcrawlable_meta(text): """ >>> _has_ajaxcrawlable_meta('') diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 0bcdd3495..4d95eb847 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -5,6 +5,7 @@ import logging from scrapy.utils.job import job_dir from scrapy.utils.request import referer_str, request_fingerprint + class BaseDupeFilter(object): @classmethod diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 96949bdd9..7c4bb3d00 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -7,10 +7,12 @@ new exceptions here without documenting them there. # Internal + class NotConfigured(Exception): """Indicates a missing configuration situation""" pass + class _InvalidOutput(TypeError): """ Indicates an invalid value has been returned by a middleware's processing method. @@ -18,15 +20,19 @@ class _InvalidOutput(TypeError): """ pass + # HTTP and crawling + class IgnoreRequest(Exception): """Indicates a decision was made not to process a request""" + class DontCloseSpider(Exception): """Request the spider not to be closed yet""" pass + class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" @@ -34,30 +40,37 @@ class CloseSpider(Exception): super(CloseSpider, self).__init__() self.reason = reason + # Items + class DropItem(Exception): """Drop item from the item pipeline""" pass + class NotSupported(Exception): """Indicates a feature or method is not supported""" pass + # Commands + class UsageError(Exception): """To indicate a command-line usage error""" def __init__(self, *a, **kw): self.print_help = kw.pop('print_help', True) super(UsageError, self).__init__(*a, **kw) + class ScrapyDeprecationWarning(Warning): """Warning category for deprecated features, since the default DeprecationWarning is silenced on Python 2.7+ """ pass + class ContractFail(AssertionError): """Error raised in case of a failing contract""" pass diff --git a/scrapy/extension.py b/scrapy/extension.py index e39e456fa..050b87e5f 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list + class ExtensionManager(MiddlewareManager): component_name = 'extension' diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 2220cbd8f..8ba770ec0 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -5,6 +5,7 @@ from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir + class SpiderState(object): """Store and load spider state during a scraping job""" diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index bd3559fbb..7eed052c2 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst from scrapy.http.response.text import TextResponse + class HtmlResponse(TextResponse): pass diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index 1df33fee5..abf474a2f 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst from scrapy.http.response.text import TextResponse + class XmlResponse(TextResponse): pass diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index d48babc3c..1896ec31e 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,5 +1,6 @@ from zope.interface import Interface + class ISpiderLoader(Interface): def from_settings(settings): diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py index 916524947..42f8de636 100644 --- a/scrapy/loader/common.py +++ b/scrapy/loader/common.py @@ -3,6 +3,7 @@ from functools import partial from scrapy.utils.python import get_func_args + def wrap_loader_context(function, context): """Wrap functions that receive loader_context to contain the context "pre-loaded" and expose a interface that receives only one argument diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 2ef8786d0..aa1bfb77f 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -7,6 +7,7 @@ See documentation in docs/item-pipeline.rst from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list + class ItemPipelineManager(MiddlewareManager): component_name = 'item pipeline' diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 0aaced7e4..4df949015 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -7,6 +7,7 @@ from scrapy.utils.datatypes import LocalCache dnscache = LocalCache(10000) + class CachingThreadedResolver(ThreadedResolver): def __init__(self, reactor, cache_size, timeout): super(CachingThreadedResolver, self).__init__(reactor) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 95a8c09b8..f0f9c59dc 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -5,8 +5,10 @@ from six import with_metaclass from scrapy.utils.python import to_unicode + logger = logging.getLogger(__name__) + def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): try: if to_native_str_type: @@ -23,6 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): robotstxt_body = '' return robotstxt_body + class RobotParser(with_metaclass(ABCMeta)): @classmethod @abstractmethod diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 2e9981556..a26e84d38 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,6 +1,7 @@ from functools import wraps from collections import OrderedDict + def _embed_ipython_shell(namespace={}, banner=''): """Start an IPython Shell""" try: @@ -23,6 +24,7 @@ def _embed_ipython_shell(namespace={}, banner=''): shell() return wrapper + def _embed_bpython_shell(namespace={}, banner=''): """Start a bpython shell""" import bpython @@ -31,6 +33,7 @@ def _embed_bpython_shell(namespace={}, banner=''): bpython.embed(locals_=namespace, banner=banner) return wrapper + def _embed_ptpython_shell(namespace={}, banner=''): """Start a ptpython shell""" import ptpython.repl @@ -40,6 +43,7 @@ def _embed_ptpython_shell(namespace={}, banner=''): ptpython.repl.embed(locals=namespace) return wrapper + def _embed_standard_shell(namespace={}, banner=''): """Start a standard python shell""" import code @@ -55,6 +59,7 @@ def _embed_standard_shell(namespace={}, banner=''): code.interact(banner=banner, local=namespace) return wrapper + DEFAULT_PYTHON_SHELLS = OrderedDict([ ('ptpython', _embed_ptpython_shell), ('ipython', _embed_ipython_shell), @@ -62,6 +67,7 @@ DEFAULT_PYTHON_SHELLS = OrderedDict([ ('python', _embed_standard_shell), ]) + def get_shell_embed_func(shells=None, known_shells=None): """Return the first acceptable shell-embed function from a given list of shell names. @@ -79,6 +85,7 @@ def get_shell_embed_func(shells=None, known_shells=None): except ImportError: continue + def start_python_console(namespace=None, banner='', shells=None): """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 38bee1a6c..2e2c7adc1 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -34,6 +34,7 @@ def defers(func): return defer.maybeDeferred(func, *a, **kw) return wrapped + def inthread(func): """Decorator to call a function in a thread and return a deferred with the result diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 69d621830..c5916c21c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -7,6 +7,7 @@ from twisted.python import failure from scrapy.exceptions import IgnoreRequest + def defer_fail(_failure): """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -18,6 +19,7 @@ def defer_fail(_failure): reactor.callLater(0.1, d.errback, _failure) return d + def defer_succeed(result): """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -29,6 +31,7 @@ def defer_succeed(result): reactor.callLater(0.1, d.callback, result) return d + def defer_result(result): if isinstance(result, defer.Deferred): return result @@ -37,6 +40,7 @@ def defer_result(result): else: return defer_succeed(result) + def mustbe_deferred(f, *args, **kw): """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop @@ -53,6 +57,7 @@ def mustbe_deferred(f, *args, **kw): else: return defer_result(result) + def parallel(iterable, count, callable, *args, **named): """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -63,6 +68,7 @@ def parallel(iterable, count, callable, *args, **named): work = (callable(elem, *args, **named) for elem in iterable) return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + def process_chain(callbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() @@ -71,6 +77,7 @@ def process_chain(callbacks, input, *a, **kw): d.callback(input) return d + def process_chain_both(callbacks, errbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() @@ -83,6 +90,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): d.callback(input) return d + def process_parallel(callbacks, input, *a, **kw): """Return a Deferred with the output of all successful calls to the given callbacks @@ -92,6 +100,7 @@ def process_parallel(callbacks, input, *a, **kw): d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d + def iter_errback(iterable, errback, *a, **kw): """Wraps an iterable calling an errback if an error is caught while iterating it. diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index f6a6c4645..91ebdae11 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -6,6 +6,7 @@ from __future__ import print_function import sys from pprint import pformat as pformat_ + def _colorize(text, colorize=True): if not colorize or not sys.stdout.isatty(): return text @@ -17,8 +18,10 @@ def _colorize(text, colorize=True): except ImportError: return text + def pformat(obj, *args, **kwargs): return _colorize(pformat_(obj), kwargs.pop('colorize', True)) + def pprint(obj, *args, **kwargs): print(pformat(obj, *args, **kwargs)) diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 11dd36d91..2c20b5c88 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -3,6 +3,7 @@ from __future__ import print_function from time import time # used in global tests code + def get_engine_status(engine): """Return a report of the current engine status""" tests = [ @@ -32,6 +33,7 @@ def get_engine_status(engine): return checks + def format_engine_status(engine=None): checks = get_engine_status(engine) s = "Execution engine status\n\n" @@ -41,5 +43,6 @@ def format_engine_status(engine=None): return s + def print_engine_status(engine): print(format_engine_status(engine)) diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 9eca6a4da..91d2439a9 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,6 +1,7 @@ from ftplib import error_perm from posixpath import dirname + def ftp_makedirs_cwd(ftp, path, first_call=True): """Set the current directory of the FTP connection given in the ``ftp`` argument (as a ftplib.FTP object), creating all parent directories if they diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index f41e62fe3..9672e28da 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -45,6 +45,7 @@ def gunzip(data): _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search + @deprecated def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index b4c929b0e..b2be0a901 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -4,7 +4,10 @@ import weakref from six.moves.urllib.parse import urlparse + _urlparse_cache = weakref.WeakKeyDictionary() + + def urlparse_cached(request_or_response): """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 389fde73a..4f1e601fc 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,5 +1,6 @@ import os + def job_dir(settings): path = settings['JOBDIR'] if path and not os.path.exists(path): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 663a8ebaa..a4201bb04 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -165,6 +165,7 @@ def memoizemethod_noargs(method): _BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} + @deprecated("scrapy.utils.python.binary_is_text") def isbinarytext(text): """ This function is deprecated. diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 83186a372..b4b5f0645 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,5 +1,6 @@ from twisted.internet import reactor, error + def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" assert len(portrange) <= 2, "invalid portrange: %s" % portrange diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 63d0ae772..0fce5a2e1 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -16,6 +16,8 @@ from scrapy.utils.httpobj import urlparse_cached _fingerprint_cache = weakref.WeakKeyDictionary() + + def request_fingerprint(request, include_headers=None, keep_fragments=False): """ Return the request fingerprint. diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index feab07431..29fdaaf2c 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -13,6 +13,8 @@ from w3lib import html _baseurl_cache = weakref.WeakKeyDictionary() + + def get_base_url(response): """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: @@ -23,6 +25,8 @@ def get_base_url(response): _metaref_cache = weakref.WeakKeyDictionary() + + def get_meta_refresh(response, ignore_tags=('script', 'noscript')): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 94b24f67e..bf4973fbf 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -28,6 +28,7 @@ def iter_spider_classes(module): getattr(obj, 'name', None): yield obj + def spidercls_for_request(spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False): """Return a spider class that handles the given Request. diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 615372fc8..96ff4b09b 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -19,6 +19,8 @@ def render_templatefile(path, **kwargs): CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') + + def string_camelcase(string): """ Convert a word to its CamelCase version and remove invalid chars diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4b935c51b..9754366df 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -32,6 +32,7 @@ def skip_if_no_boto(): except NotConfigured as e: raise SkipTest(e) + def get_s3_content_and_delete(bucket, path, with_key=False): """ Get content from s3 key, and delete key afterwards. """ @@ -51,6 +52,7 @@ def get_s3_content_and_delete(bucket, path, with_key=False): bucket.delete_key(path) return (content, key) if with_key else content + def get_gcs_content_and_delete(bucket, path): from google.cloud import storage client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) @@ -61,6 +63,7 @@ def get_gcs_content_and_delete(bucket, path): bucket.delete_blob(path) return content, acl, blob + def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -72,12 +75,14 @@ def get_crawler(spidercls=None, settings_dict=None): runner = CrawlerRunner(settings_dict) return runner.create_crawler(spidercls or Spider) + def get_pythonpath(): """Return a PYTHONPATH suitable to use in processes so that they find this installation of Scrapy""" scrapy_path = import_module('scrapy').__path__[0] return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '') + def get_testenv(): """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. @@ -86,6 +91,7 @@ def get_testenv(): env['PYTHONPATH'] = get_pythonpath() return env + def assert_samelines(testcase, text1, text2, msg=None): """Asserts text1 and text2 have the same lines, ignoring differences in line endings between platforms diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 48484b303..b0737d3d5 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -27,5 +27,5 @@ def scrapy_components_versions(): ("Python", sys.version.replace("\n", "- ")), ("pyOpenSSL", get_openssl_version()), ("cryptography", cryptography.__version__), - ("Platform", platform.platform()), + ("Platform", platform.platform()), ] diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 431428331..75c74daf5 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -13,6 +13,7 @@ error = KeyError _DATABASES = collections.defaultdict(DummyDB) + def open(file, flag='r', mode=0o666): """Open or create a dummy database compatible. diff --git a/tests/pipelines.py b/tests/pipelines.py index 7e2895a5c..d7d3b5259 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -2,6 +2,7 @@ Some pipelines used for testing """ + class ZeroDivisionErrorPipeline(object): def open_spider(self, spider): diff --git a/tests/spiders.py b/tests/spiders.py index 7816bf7c7..2487ecc22 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -16,6 +16,7 @@ class MockServerSpider(Spider): super(MockServerSpider, self).__init__(*args, **kwargs) self.mockserver = mockserver + class MetaSpider(MockServerSpider): name = 'meta' diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index 28456b55d..c64e87d81 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -1,5 +1,6 @@ """A test extension used to check the settings loading order""" + class TestExtension(object): def __init__(self, settings): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index b134beb88..b7035fdff 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -12,6 +12,7 @@ def _textmode(bstr): and reading from it in text mode""" return to_unicode(bstr).replace(os.linesep, '\n') + class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 03bf2ffcf..e31ccd9b5 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,6 +1,7 @@ from importlib import import_module from twisted.trial import unittest + class ScrapyUtilsTest(unittest.TestCase): def test_required_openssl_version(self): try: diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 493691ea4..5a56c9db2 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -5,8 +5,10 @@ from scrapy.spiders import Spider from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler + __doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl'] + class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True}) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9d863b6e3..00e6c685e 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -149,6 +149,7 @@ class FilesystemStorageTest(DefaultStorageTest): storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage' + class FilesystemStorageGzipTest(FilesystemStorageTest): def _get_settings(self, **new_settings): diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index d7eb98c97..e4b0bdf83 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -12,6 +12,7 @@ from scrapy.utils.job import job_dir from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider + class FromCrawlerRFPDupeFilter(RFPDupeFilter): @classmethod diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 69d906fbf..c83cf3b66 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -3,6 +3,7 @@ import copy from scrapy.http import Headers + class HeadersTest(unittest.TestCase): def assertSortedEqual(self, first, second, msg=None): diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index b4ea30bb7..0724d1807 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -118,6 +118,7 @@ class DropSomeItemsPipeline(object): else: self.drop = True + class ShowOrSkipMessagesTestCase(TwistedTestCase): def setUp(self): self.mockserver = MockServer() diff --git a/tests/test_mail.py b/tests/test_mail.py index b139e98d8..ddb0f1e70 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -6,6 +6,7 @@ from email.charset import Charset from scrapy.mail import MailSender + class MailSenderTest(unittest.TestCase): def test_send(self): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index af9b43d61..ebf817c7e 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -4,6 +4,7 @@ from scrapy.settings import Settings from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager + class M1(object): def open_spider(self, spider): @@ -15,6 +16,7 @@ class M1(object): def process(self, response, request, spider): pass + class M2(object): def open_spider(self, spider): @@ -25,6 +27,7 @@ class M2(object): pass + class M3(object): def process(self, response, request, spider): @@ -54,6 +57,7 @@ class TestMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process'): self.methods['process'].append(mw.process) + class MiddlewareManagerTest(unittest.TestCase): def test_init(self): diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index f89042b3d..d5a3371ab 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -4,6 +4,7 @@ from scrapy.responsetypes import responsetypes from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Headers + class ResponseTypesTest(unittest.TestCase): def test_from_filename(self): diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index cd7480e33..080507276 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -19,6 +19,7 @@ def rerp_available(): return False return True + def protego_available(): # check if protego parser is installed try: @@ -27,6 +28,7 @@ def protego_available(): return False return True + class BaseRobotParserTest: def _setUp(self, parser_cls): self.parser_cls = parser_cls diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 106da798c..d8be6e277 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -109,6 +109,7 @@ class SpiderLoaderTest(unittest.TestCase): spiders = spider_loader.list() self.assertEqual(spiders, []) + class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py index 35b71870a..dbd1fb123 100644 --- a/tests/test_spiderloader/test_spiders/nested/spider4.py +++ b/tests/test_spiderloader/test_spiders/nested/spider4.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider4(Spider): name = "spider4" allowed_domains = ['spider4.com'] diff --git a/tests/test_spiderloader/test_spiders/spider0.py b/tests/test_spiderloader/test_spiders/spider0.py index 75a90794e..af679dbd6 100644 --- a/tests/test_spiderloader/test_spiders/spider0.py +++ b/tests/test_spiderloader/test_spiders/spider0.py @@ -1,4 +1,5 @@ from scrapy.spiders import Spider + class Spider0(Spider): allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/tests/test_spiderloader/test_spiders/spider1.py b/tests/test_spiderloader/test_spiders/spider1.py index 76efddc7f..6b4317a90 100644 --- a/tests/test_spiderloader/test_spiders/spider1.py +++ b/tests/test_spiderloader/test_spiders/spider1.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider1(Spider): name = "spider1" allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/tests/test_spiderloader/test_spiders/spider2.py b/tests/test_spiderloader/test_spiders/spider2.py index 0badd8437..352601863 100644 --- a/tests/test_spiderloader/test_spiders/spider2.py +++ b/tests/test_spiderloader/test_spiders/spider2.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider2(Spider): name = "spider2" allowed_domains = ["scrapy2.org", "scrapy3.org"] diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py index d406f2d4f..84998ba35 100644 --- a/tests/test_spiderloader/test_spiders/spider3.py +++ b/tests/test_spiderloader/test_spiders/spider3.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider3(Spider): name = "spider3" allowed_domains = ['spider3.com'] diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 7e4af0d4c..b97d9b675 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -9,6 +9,7 @@ from scrapy.spidermiddlewares.offsite import URLWarning from scrapy.utils.test import get_crawler import warnings + class TestOffsiteMiddleware(TestCase): def setUp(self): @@ -53,6 +54,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): out = list(self.mw.process_spider_output(res, reqs, self.spider)) self.assertEqual(out, reqs) + class TestOffsiteMiddleware3(TestOffsiteMiddleware2): def _get_spider(self): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 6f8727a15..940e31070 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -34,6 +34,7 @@ class RecoverySpider(Spider): if not response.meta.get('dont_fail'): raise TabError() + class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) @@ -50,6 +51,7 @@ class FailProcessSpiderInputMiddleware: spider.logger.info('Middleware: will raise IndexError') raise IndexError() + class ProcessSpiderInputSpiderWithoutErrback(Spider): name = 'ProcessSpiderInputSpiderWithoutErrback' custom_settings = { @@ -177,6 +179,7 @@ class GeneratorRecoverMiddleware: spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} + class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): pass @@ -247,6 +250,7 @@ class NotGeneratorRecoverMiddleware: spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] + class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): pass diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 21439c20e..a9c31a983 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -349,6 +349,7 @@ class TestSettingsCustomPolicy(TestRefererMiddleware): ] + # --- Tests using Request meta dict to set policy class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} @@ -518,14 +519,17 @@ class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} + class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} + class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} + class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): """ The empty string means "no-referrer-when-downgrade" diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 3ded5c027..d5fcf2f7f 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -7,16 +7,20 @@ from scrapy.http import Request from scrapy.loader import ItemLoader from scrapy.selector import Selector + class TestItem(Item): name = Field() + def _test_procesor(x): return x + x + class TestLoader(ItemLoader): default_item_class = TestItem name_out = staticmethod(_test_procesor) + def nonserializable_object_test(self): q = self.queue() try: @@ -35,6 +39,7 @@ def nonserializable_object_test(self): sel = Selector(text='

some text

') self.assertRaises(ValueError, q.push, sel) + class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): chunksize = 100000 @@ -53,15 +58,19 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): test_nonserializable_object = nonserializable_object_test + class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 + class ChunkSize2MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 2 + class ChunkSize3MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 3 + class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 4 @@ -100,15 +109,19 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest): self.assertEqual(r.url, r2.url) assert r2.meta['request'] is r2 + class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 1 + class ChunkSize2PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 2 + class ChunkSize3PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 3 + class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 4 diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index c2211848c..380c41367 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -14,6 +14,7 @@ try: except ImportError: ipy = False + class UtilsConsoleTestCase(unittest.TestCase): def test_get_shell_embed_func(self): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 003bb9b02..d642ed3ed 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -33,14 +33,23 @@ class MustbeDeferredTest(unittest.TestCase): steps.append(2) # add another value, that should be catched by assertEqual return dfd + def cb1(value, arg1, arg2): return "(cb1 %s %s %s)" % (value, arg1, arg2) + + def cb2(value, arg1, arg2): return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2)) + + def cb3(value, arg1, arg2): return "(cb3 %s %s %s)" % (value, arg1, arg2) + + def cb_fail(value, arg1, arg2): return Failure(TypeError()) + + def eb1(failure, arg1, arg2): return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2) diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py index 2524153ea..f9af4bf87 100644 --- a/tests/test_utils_http.py +++ b/tests/test_utils_http.py @@ -2,6 +2,7 @@ import unittest from scrapy.utils.http import decode_chunked_transfer + class ChunkedTest(unittest.TestCase): def test_decode_chunked_transfer(self): diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 4f9f7a370..2c3965bbc 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -4,6 +4,7 @@ from six.moves.urllib.parse import urlparse from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached + class HttpobjUtilsTest(unittest.TestCase): def test_urlparse_cached(self): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 2d845697e..f16ef8110 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -235,6 +235,7 @@ class LxmlXmliterTestCase(XmliterTestCase): i = self.xmliter(42, 'product') self.assertRaises(TypeError, next, i) + class UtilsCsvTestCase(unittest.TestCase): sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds') sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv') diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 625a32048..3da95b95a 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -4,6 +4,7 @@ from scrapy.http import Request from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \ request_authenticate, request_httprepr + class UtilsRequestTest(unittest.TestCase): def test_request_fingerprint(self): diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 62edd420d..16b7c5c68 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -66,6 +66,7 @@ class SendCatchLogDeferredTest2(SendCatchLogTest): def _get_result(self, signal, *a, **kw): return send_catch_log_deferred(signal, *a, **kw) + class SendCatchLogTest2(unittest.TestCase): def test_error_logged_if_deferred_not_supported(self): diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 716bb44eb..db323ab31 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -2,6 +2,7 @@ import unittest from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots + class SitemapTest(unittest.TestCase): def test_sitemap(self): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index d9de1ce77..edeeacc80 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -9,12 +9,15 @@ from scrapy.spiders import CrawlSpider class MyBaseSpider(CrawlSpider): pass # abstract spider + class MySpider1(MyBaseSpider): name = 'myspider1' + class MySpider2(MyBaseSpider): name = 'myspider2' + class UtilsSpidersTestCase(unittest.TestCase): def test_iterate_spider_output(self): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index e6588055c..93addc082 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -187,6 +187,7 @@ class AddHttpIfNoScheme(unittest.TestCase): class GuessSchemeTest(unittest.TestCase): pass + def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) @@ -195,6 +196,7 @@ def create_guess_scheme_t(args): args[0], url, args[1]) return do_expected + def create_skipped_scheme_t(args): def do_expected(self): raise unittest.SkipTest(args[2])