diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8d4d74bc5..3c1c8f891 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.2.0 +current_version = 2.3.0 commit = True tag = True tag_name = {new_version} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfbdf4208 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/sample_data/** binary diff --git a/.travis.yml b/.travis.yml index db720b918..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,16 +19,10 @@ matrix: python: 3.8 - env: TOXENV=pinned - python: 3.5.2 + python: 3.6.1 - env: TOXENV=asyncio-pinned - python: 3.5.2 # We use additional code to support 3.5.3 and earlier - - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 - - - env: TOXENV=py - python: 3.5 - - env: TOXENV=asyncio - python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 + python: 3.6.1 + - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - env: TOXENV=py python: 3.6 @@ -50,7 +44,7 @@ install: - | if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" diff --git a/README.rst b/README.rst index 0e3939e9b..a8f2ba52b 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ including a list of features. Requirements ============ -* Python 3.5.2+ +* Python 3.6+ * Works on Linux, Windows, macOS, BSD Install diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 7fd636864..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,25 +0,0 @@ -platform: x86 -version: '{branch}-{build}' -environment: - matrix: - - PYTHON: "C:\\Python36" - TOX_ENV: py36 - -branches: - only: - - master - - /d+\.\d+\.\d+[\w\-]*$/ - -install: - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%" - - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE" - - "pip install -U tox" - -build: false -skip_tags: true -test_script: - - "tox -e %TOX_ENV%" - -cache: - - '%LOCALAPPDATA%\pip\cache' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 710e42090..c03e258c7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,11 +4,9 @@ pool: vmImage: 'windows-latest' strategy: matrix: - Python35: - python.version: '3.5' - TOXENV: windows-pinned Python36: python.version: '3.6' + TOXENV: windows-pinned Python37: python.version: '3.7' Python38: diff --git a/conftest.py b/conftest.py index b39d644a5..be97b7714 100644 --- a/conftest.py +++ b/conftest.py @@ -14,8 +14,6 @@ collect_ignore = [ *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), - # Py36-only parts of respective tests - *_py_files("tests/py36"), ] for line in open('tests/ignores.txt'): diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 192123473..640660943 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -17,7 +17,7 @@ class SettingsListDirective(Directive): def is_setting_index(node): if node.tagname == 'index': # index entries for setting directives look like: - # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')] + # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] entry_type, info, refid = node['entries'][0][:3] return entry_type == 'pair' and info.endswith('; setting') return False diff --git a/docs/conf.py b/docs/conf.py index 427c79481..27d2b5dff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,7 +49,7 @@ master_doc = 'index' # General information about the project. project = 'Scrapy' -copyright = '2008–{}, Scrapy developers'.format(datetime.now().year) +copyright = f'2008–{datetime.now().year}, Scrapy developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/contributing.rst b/docs/contributing.rst index 7b901dd00..675f55c38 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -108,6 +108,11 @@ Well-written patches should: tox -e docs-coverage +* if you are removing deprecated code, first make sure that at least 1 year + (12 months) has passed since the release that introduced the deprecation. + See :ref:`deprecation-policy`. + + .. _submitting-patches: Submitting patches @@ -194,6 +199,17 @@ In any case, if something is covered in a docstring, use the documentation instead of duplicating the docstring in files within the ``docs/`` directory. +Documentation updates that cover new or modified features must use Sphinx’s +:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use +``VERSION`` as version, we will replace it with the actual version right before +the corresponding release. When we release a new major or minor version of +Scrapy, we remove these directives if they are older than 3 years. + +Documentation about deprecated features must be removed as those features are +deprecated, so that new readers do not run into it. New deprecations and +deprecation removals are documented in the :ref:`release notes `. + + Tests ===== diff --git a/docs/faq.rst b/docs/faq.rst index ea2c8216f..9346ec358 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -236,15 +236,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file? To dump into a JSON file:: - scrapy crawl myspider -o items.json + scrapy crawl myspider -O items.json To dump into a CSV file:: - scrapy crawl myspider -o items.csv + scrapy crawl myspider -O items.csv To dump into a XML file:: - scrapy crawl myspider -o items.xml + scrapy crawl myspider -O items.xml For more information see :ref:`topics-feed-exports` diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 6d65ae2ee..8b4240bf6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,8 +9,8 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.5.2+, either the CPython implementation (default) or -the PyPy 5.9+ implementation (see :ref:`python:implementations`). +Scrapy requires Python 3.6+, either the CPython implementation (default) or +the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). Installing Scrapy diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 01986b594..dd80c7bd0 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination:: if next_page is not None: yield response.follow(next_page, self.parse) - Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider quotes_spider.py -o quotes.json + scrapy runspider quotes_spider.py -o quotes.jl +When this finishes you will have in the ``quotes.jl`` file a list of the +quotes in JSON Lines format, containing text and author, looking like this:: -When this finishes you will have in the ``quotes.json`` file a list of the -quotes in JSON format, containing text and author, looking like this (reformatted -here for better readability):: - - [{ - "author": "Jane Austen", - "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" - }, - { - "author": "Groucho Marx", - "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" - }, - { - "author": "Steve Martin", - "text": "\u201cA day without sunshine is like, you know, night.\u201d" - }, - ...] + {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} + {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} + {"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"} + ... What just happened? diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 5f35dc936..9270ff42c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -101,10 +101,10 @@ This is the code for our first Spider. Save it in a file named def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) - self.log('Saved file %s' % filename) + self.log(f'Saved file {filename}') As you can see, our Spider subclasses :class:`scrapy.Spider ` @@ -190,7 +190,7 @@ for your spider:: def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) @@ -405,8 +405,6 @@ to get all of them: from sys import version_info -.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output") - Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary: @@ -464,16 +462,15 @@ Storing the scraped data The simplest way to store the scraped data is by using :ref:`Feed exports `, with the following command:: - scrapy crawl quotes -o quotes.json + scrapy crawl quotes -O quotes.json That will generate an ``quotes.json`` file containing all scraped items, serialized in `JSON`_. -For historic reasons, Scrapy appends to a given file instead of overwriting -its contents. If you run this command twice without removing the file -before the second time, you'll end up with a broken JSON file. - -You can also use other formats, like `JSON Lines`_:: +The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead +to append new content to any existing file. However, appending to a JSON file +makes the file contents invalid JSON. When appending to a file, consider +using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jl @@ -704,7 +701,7 @@ Using spider arguments You can provide command line arguments to your spiders by using the ``-a`` option when running them:: - scrapy crawl quotes -o quotes-humor.json -a tag=humor + scrapy crawl quotes -O quotes-humor.json -a tag=humor These arguments are passed to the Spider's ``__init__`` method and become spider attributes by default. diff --git a/docs/news.rst b/docs/news.rst index 80d130e4a..850b323ef 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,139 @@ Release notes ============= +.. _release-2.3.0: + +Scrapy 2.3.0 (2020-08-04) +------------------------- + +Highlights: + +* :ref:`Feed exports ` now support :ref:`Google Cloud + Storage ` as a storage backend + +* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver + output items in batches of up to the specified number of items. + + It also serves as a workaround for :ref:`delayed file delivery + `, which causes Scrapy to only start item delivery + after the crawl has finished when using certain storage backends + (:ref:`S3 `, :ref:`FTP `, + and now :ref:`GCS `). + +* The base implementation of :ref:`item loaders ` has been + moved into a separate library, :doc:`itemloaders `, + allowing usage from outside Scrapy and a separate release schedule + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* Removed the following classes and their parent modules from + ``scrapy.linkextractors``: + + * ``htmlparser.HtmlParserLinkExtractor`` + * ``regex.RegexLinkExtractor`` + * ``sgml.BaseSgmlLinkExtractor`` + * ``sgml.SgmlLinkExtractor`` + + Use + :class:`LinkExtractor ` + instead (:issue:`4356`, :issue:`4679`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated + (:issue:`4683`) + + +New features +~~~~~~~~~~~~ + +* :ref:`Feed exports ` support :ref:`Google Cloud + Storage ` (:issue:`685`, :issue:`3608`) + +* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries + (:issue:`4250`, :issue:`4434`) + +* The :command:`parse` command now allows specifying an output file + (:issue:`4317`, :issue:`4377`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support + ``--data-raw`` (:issue:`4612`) + +* A ``parse`` callback may now be used in built-in spider subclasses, such + as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`, + :issue:`781`, :issue:`4254` ) + + +Bug fixes +~~~~~~~~~ + +* Fixed the :ref:`CSV exporting ` of + :ref:`dataclass items ` and :ref:`attr.s items + ` (:issue:`4667`, :issue:`4668`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request + method to ``POST`` when a request body is specified and no request method + is specified (:issue:`4612`) + +* The processing of ANSI escape sequences in enabled in Windows 10.0.14393 + and later, where it is required for colored output (:issue:`4393`, + :issue:`4403`) + + +Documentation +~~~~~~~~~~~~~ + +* Updated the `OpenSSL cipher list format`_ link in the documentation about + the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`) + +* Simplified the code example in :ref:`topics-loaders-dataclass` + (:issue:`4652`) + +.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* The base implementation of :ref:`item loaders ` has been + moved into :doc:`itemloaders ` (:issue:`4005`, + :issue:`4516`) + +* Fixed a silenced error in some scheduler tests (:issue:`4644`, + :issue:`4645`) + +* Renewed the localhost certificate used for SSL tests (:issue:`4650`) + +* Removed cookie-handling code specific to Python 2 (:issue:`4682`) + +* Stopped using Python 2 unicode literal syntax (:issue:`4704`) + +* Stopped using a backlash for line continuation (:issue:`4673`) + +* Removed unneeded entries from the MyPy exception list (:issue:`4690`) + +* Automated tests now pass on Windows as part of our continuous integration + system (:issue:`4458`) + +* Automated tests now pass on the latest PyPy version for supported Python + versions in our continuous integration system (:issue:`4504`) + + +.. _release-2.2.1: + +Scrapy 2.2.1 (2020-07-17) +------------------------- + +* The :command:`startproject` command no longer makes unintended changes to + the permissions of files in the destination folder, such as removing + execution permissions (:issue:`4662`, :issue:`4666`) + + .. _release-2.2.0: Scrapy 2.2.0 (2020-06-24) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 52509ffdf..445b2979f 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -4,8 +4,6 @@ Core API ======== -.. versionadded:: 0.15 - This section documents the Scrapy core API, and it's intended for developers of extensions and middlewares. diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 038a459fd..bfb430d52 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -26,3 +26,15 @@ reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`:: install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to +use it instead of the default asyncio event loop. + + + diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 4317019fc..8e6aae65c 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies. AUTOTHROTTLE_TARGET_CONCURRENCY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 1.1 - Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 99469ebf1..b01a66188 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -4,8 +4,6 @@ Benchmarking ============ -.. versionadded:: 0.17 - Scrapy comes with a simple benchmarking suite that spawns a local HTTP server and crawls it at the maximum possible speed. The goal of this benchmarking is to get an idea of how Scrapy performs in your hardware, in order to have a diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index f9bb4f0f9..3c2763917 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,8 +6,6 @@ Command line tool ================= -.. versionadded:: 0.10 - Scrapy is controlled through the ``scrapy`` command-line tool, to be referred here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -497,6 +495,8 @@ Supported options: * ``--output`` or ``-o``: dump scraped items to a file + .. versionadded:: 2.3 + .. skip: start Usage example:: @@ -568,8 +568,6 @@ and Platform info, which is useful for bug reports. bench ----- -.. versionadded:: 0.17 - * Syntax: ``scrapy bench`` * Requires project: *no* diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index b8b3078c4..e61421bf1 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -4,8 +4,6 @@ Spiders Contracts ================= -.. versionadded:: 0.15 - Testing spiders can get particularly annoying and while nothing prevents you from writing unit tests the task gets cumbersome quickly. Scrapy offers an integrated way of testing your spiders by the means of contracts. @@ -81,7 +79,7 @@ override three methods: .. class:: Contract(method, *args) :param method: callback function to which the contract is associated - :type method: function + :type method: collections.abc.Callable :param args: list of arguments passed into the docstring (whitespace separated) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a0952d323..3b1549bd3 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -17,19 +17,14 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.http.Request` callbacks. - The following are known caveats of the current implementation that we aim - to address in future versions of Scrapy: - - - The callback output is not processed until the whole callback finishes. + .. note:: The callback output is not processed until the whole callback + finishes. As a side effect, if the callback raises an exception, none of its output is processed. - - Because `asynchronous generators were introduced in Python 3.6`_, you - can only use ``yield`` if you are using Python 3.6 or later. - - If you need to output multiple items or requests and you are using - Python 3.5, return an iterable (e.g. a list) instead. + This is a known caveat of the current implementation that we aim to + address in a future version of Scrapy. - The :meth:`process_item` method of :ref:`item pipelines `. @@ -44,8 +39,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. -.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/ - Usage ===== diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 4e87a00f2..c83b1a9d9 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping ================================================= Here is a general guide on how to use your browser's Developer Tools -to ease the scraping process. Today almost all browsers come with +to ease the scraping process. Today almost all browsers come with built in `Developer Tools`_ and although we will use Firefox in this -guide, the concepts are applicable to any other browser. +guide, the concepts are applicable to any other browser. In this guide we'll introduce the basic tools to use from a browser's Developer Tools by scraping `quotes.toscrape.com`_. @@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things: Inspecting a website ==================== -By far the most handy feature of the Developer Tools is the `Inspector` -feature, which allows you to inspect the underlying HTML code of -any webpage. To demonstrate the Inspector, let's look at the +By far the most handy feature of the Developer Tools is the `Inspector` +feature, which allows you to inspect the underlying HTML code of +any webpage. To demonstrate the Inspector, let's look at the `quotes.toscrape.com`_-site. On the site we have a total of ten quotes from various authors with specific -tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes -on this page, without any meta-information about authors, tags, etc. +tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes +on this page, without any meta-information about authors, tags, etc. -Instead of viewing the whole source code for the page, we can simply right click +Instead of viewing the whole source code for the page, we can simply right click on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`. In it you should see something like this: @@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] -Adding ``text()`` at the end we are able to extract the first quote with this +Adding ``text()`` at the end we are able to extract the first quote with this basic selector. But this XPath is not really that clever. All it does is -go down a desired path in the source code starting from ``html``. So let's -see if we can refine our XPath a bit: +go down a desired path in the source code starting from ``html``. So let's +see if we can refine our XPath a bit: -If we check the `Inspector` again we'll see that directly beneath our -expanded ``div`` tag we have nine identical ``div`` tags, each with the -same attributes as our first. If we expand any of them, we'll see the same +If we check the `Inspector` again we'll see that directly beneath our +expanded ``div`` tag we have nine identical ``div`` tags, each with the +same attributes as our first. If we expand any of them, we'll see the same structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can -expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and +expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and see each quote: .. code-block:: html @@ -121,7 +121,7 @@ see each quote: With this knowledge we can refine our XPath: Instead of a path to follow, -we'll simply select all ``span`` tags with the ``class="text"`` by using +we'll simply select all ``span`` tags with the ``class="text"`` by using the `has-class-extension`_: >>> response.xpath('//span[has-class("text")]/text()').getall() @@ -130,45 +130,45 @@ the `has-class-extension`_: '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', ...] -And with one simple, cleverer XPath we are able to extract all quotes from -the page. We could have constructed a loop over our first XPath to increase -the number of the last ``div``, but this would have been unnecessarily +And with one simple, cleverer XPath we are able to extract all quotes from +the page. We could have constructed a loop over our first XPath to increase +the number of the last ``div``, but this would have been unnecessarily complex and by simply constructing an XPath with ``has-class("text")`` -we were able to extract all quotes in one line. +we were able to extract all quotes in one line. -The `Inspector` has a lot of other helpful features, such as searching in the +The `Inspector` has a lot of other helpful features, such as searching in the source code or directly scrolling to an element you selected. Let's demonstrate -a use case: +a use case: -Say you want to find the ``Next`` button on the page. Type ``Next`` into the -search bar on the top right of the `Inspector`. You should get two results. -The first is a ``li`` tag with the ``class="next"``, the second the text +Say you want to find the ``Next`` button on the page. Type ``Next`` into the +search bar on the top right of the `Inspector`. You should get two results. +The first is a ``li`` tag with the ``class="next"``, the second the text of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``. If you hover over the tag, you'll see the button highlighted. From here -we could easily create a :ref:`Link Extractor ` to -follow the pagination. On a simple site such as this, there may not be +we could easily create a :ref:`Link Extractor ` to +follow the pagination. On a simple site such as this, there may not be the need to find an element visually but the ``Scroll into View`` function -can be quite useful on complex sites. +can be quite useful on complex sites. Note that the search bar can also be used to search for and test CSS -selectors. For example, you could search for ``span.text`` to find -all quote texts. Instead of a full text search, this searches for -exactly the ``span`` tag with the ``class="text"`` in the page. +selectors. For example, you could search for ``span.text`` to find +all quote texts. Instead of a full text search, this searches for +exactly the ``span`` tag with the ``class="text"`` in the page. .. _topics-network-tool: The Network-tool ================ While scraping you may come across dynamic webpages where some parts -of the page are loaded dynamically through multiple requests. While -this can be quite tricky, the `Network`-tool in the Developer Tools +of the page are loaded dynamically through multiple requests. While +this can be quite tricky, the `Network`-tool in the Developer Tools greatly facilitates this task. To demonstrate the Network-tool, let's -take a look at the page `quotes.toscrape.com/scroll`_. +take a look at the page `quotes.toscrape.com/scroll`_. -The page is quite similar to the basic `quotes.toscrape.com`_-page, -but instead of the above-mentioned ``Next`` button, the page -automatically loads new quotes when you scroll to the bottom. We -could go ahead and try out different XPaths directly, but instead +The page is quite similar to the basic `quotes.toscrape.com`_-page, +but instead of the above-mentioned ``Next`` button, the page +automatically loads new quotes when you scroll to the bottom. We +could go ahead and try out different XPaths directly, but instead we'll check another quite useful command from the Scrapy shell: .. skip: next @@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell: (...) >>> view(response) -A browser window should open with the webpage but with one -crucial difference: Instead of the quotes we just see a greenish -bar with the word ``Loading...``. +A browser window should open with the webpage but with one +crucial difference: Instead of the quotes we just see a greenish +bar with the word ``Loading...``. .. image:: _images/network_01.png :width: 777 @@ -189,21 +189,21 @@ bar with the word ``Loading...``. :alt: Response from quotes.toscrape.com/scroll The ``view(response)`` command let's us view the response our -shell or later our spider receives from the server. Here we see -that some basic template is loaded which includes the title, +shell or later our spider receives from the server. Here we see +that some basic template is loaded which includes the title, the login-button and the footer, but the quotes are missing. This tells us that the quotes are being loaded from a different request -than ``quotes.toscrape/scroll``. +than ``quotes.toscrape/scroll``. -If you click on the ``Network`` tab, you will probably only see -two entries. The first thing we do is enable persistent logs by -clicking on ``Persist Logs``. If this option is disabled, the +If you click on the ``Network`` tab, you will probably only see +two entries. The first thing we do is enable persistent logs by +clicking on ``Persist Logs``. If this option is disabled, the log is automatically cleared each time you navigate to a different -page. Enabling this option is a good default, since it gives us -control on when to clear the logs. +page. Enabling this option is a good default, since it gives us +control on when to clear the logs. If we reload the page now, you'll see the log get populated with six -new requests. +new requests. .. image:: _images/network_02.png :width: 777 @@ -212,31 +212,31 @@ new requests. Here we see every request that has been made when reloading the page and can inspect each request and its response. So let's find out -where our quotes are coming from: +where our quotes are coming from: -First click on the request with the name ``scroll``. On the right +First click on the request with the name ``scroll``. On the right you can now inspect the request. In ``Headers`` you'll find details about the request headers, such as the URL, the method, the IP-address, and so on. We'll ignore the other tabs and click directly on ``Response``. -What you should see in the ``Preview`` pane is the rendered HTML-code, -that is exactly what we saw when we called ``view(response)`` in the -shell. Accordingly the ``type`` of the request in the log is ``html``. -The other requests have types like ``css`` or ``js``, but what -interests us is the one request called ``quotes?page=1`` with the -type ``json``. +What you should see in the ``Preview`` pane is the rendered HTML-code, +that is exactly what we saw when we called ``view(response)`` in the +shell. Accordingly the ``type`` of the request in the log is ``html``. +The other requests have types like ``css`` or ``js``, but what +interests us is the one request called ``quotes?page=1`` with the +type ``json``. -If we click on this request, we see that the request URL is +If we click on this request, we see that the request URL is ``http://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click -on the request and open ``Open in new tab`` to get a better overview. +on the request and open ``Open in new tab`` to get a better overview. .. image:: _images/network_03.png :width: 777 :height: 375 :alt: JSON-object returned from the quotes.toscrape API -With this response we can now easily parse the JSON-object and +With this response we can now easily parse the JSON-object and also request each page to get every quote on the site:: import scrapy @@ -255,17 +255,17 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page) + url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) -This spider starts at the first page of the quotes-API. With each -response, we parse the ``response.text`` and assign it to ``data``. -This lets us operate on the JSON-object like on a Python dictionary. +This spider starts at the first page of the quotes-API. With each +response, we parse the ``response.text`` and assign it to ``data``. +This lets us operate on the JSON-object like on a Python dictionary. We iterate through the ``quotes`` and print out the ``quote["text"]``. -If the handy ``has_next`` element is ``true`` (try loading +If the handy ``has_next`` element is ``true`` (try loading `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a -page-number greater than 10), we increment the ``page`` attribute -and ``yield`` a new request, inserting the incremented page-number +page-number greater than 10), we increment the ``page`` attribute +and ``yield`` a new request, inserting the incremented page-number into our ``url``. .. _requests-from-curl: @@ -289,14 +289,16 @@ request:: "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") Alternatively, if you want to know the arguments needed to recreate that -request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs` -function to get a dictionary with the equivalent arguments. +request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs` +function to get a dictionary with the equivalent arguments: + +.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs Note that to translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. As you can see, with a few inspections in the `Network`-tool we -were able to easily replicate the dynamic requests of the scrolling +were able to easily replicate the dynamic requests of the scrolling functionality of the page. Crawling dynamic pages can be quite daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 323e553e5..06e614941 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -217,8 +217,6 @@ The following settings can be used to configure the cookie middleware: Multiple cookie sessions per spider ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 0.15 - There is support for keeping multiple cookie sessions per spider by using the :reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar (session), but you can pass an identifier to use different ones. @@ -475,8 +473,6 @@ DBM storage backend .. class:: DbmCacheStorage - .. versionadded:: 0.13 - A DBM_ storage backend is also available for the HTTP cache middleware. By default, it uses the :mod:`dbm`, but you can change it with the @@ -549,15 +545,10 @@ settings: HTTPCACHE_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.11 - Default: ``False`` Whether the HTTP cache will be enabled. -.. versionchanged:: 0.11 - Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache. - .. setting:: HTTPCACHE_EXPIRATION_SECS HTTPCACHE_EXPIRATION_SECS @@ -570,9 +561,6 @@ Expiration time for cached requests, in seconds. Cached requests older than this time will be re-downloaded. If zero, cached requests will never expire. -.. versionchanged:: 0.11 - Before 0.11, zero meant cached requests always expire. - .. setting:: HTTPCACHE_DIR HTTPCACHE_DIR @@ -589,8 +577,6 @@ project data dir. For more info see: :ref:`topics-project-structure`. HTTPCACHE_IGNORE_HTTP_CODES ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``[]`` Don't cache response with these HTTP codes. @@ -609,8 +595,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade HTTPCACHE_IGNORE_SCHEMES ^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``['file']`` Don't cache responses with these URI schemes. @@ -629,8 +613,6 @@ The class which implements the cache storage backend. HTTPCACHE_DBM_MODULE ^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``'dbm'`` The database module to use in the :ref:`DBM storage backend @@ -641,8 +623,6 @@ The database module to use in the :ref:`DBM storage backend HTTPCACHE_POLICY ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.18 - Default: ``'scrapy.extensions.httpcache.DummyPolicy'`` The class which implements the cache policy. @@ -652,8 +632,6 @@ The class which implements the cache policy. HTTPCACHE_GZIP ^^^^^^^^^^^^^^ -.. versionadded:: 1.0 - Default: ``False`` If enabled, will compress all cached data with gzip. @@ -664,8 +642,6 @@ This setting is specific to the Filesystem backend. HTTPCACHE_ALWAYS_STORE ^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``False`` If enabled, will cache pages unconditionally. @@ -684,8 +660,6 @@ responses you feed to the cache middleware. HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``[]`` List of Cache-Control directives in responses to be ignored. @@ -735,8 +709,6 @@ HttpProxyMiddleware .. module:: scrapy.downloadermiddlewares.httpproxy :synopsis: Http Proxy Middleware -.. versionadded:: 0.8 - .. reqmeta:: proxy .. class:: HttpProxyMiddleware @@ -817,8 +789,6 @@ RedirectMiddleware settings REDIRECT_ENABLED ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Redirect middleware will be enabled. @@ -860,8 +830,6 @@ MetaRefreshMiddleware settings METAREFRESH_ENABLED ^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.17 - Default: ``True`` Whether the Meta Refresh middleware will be enabled. @@ -924,8 +892,6 @@ RetryMiddleware Settings RETRY_ENABLED ^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Retry middleware will be enabled. @@ -1179,8 +1145,6 @@ AjaxCrawlMiddleware Settings AJAXCRAWL_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.21 - Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. You may want to diff --git a/docs/topics/email.rst b/docs/topics/email.rst index e347c3a35..1a2bc6330 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -62,10 +62,10 @@ rest of the framework. :type smtpport: int :param smtptls: enforce using SMTP STARTTLS - :type smtptls: boolean + :type smtptls: bool :param smtpssl: enforce using a secure SSL connection - :type smtpssl: boolean + :type smtpssl: bool .. classmethod:: from_settings(settings) @@ -79,14 +79,14 @@ rest of the framework. Send email to the given recipients. - :param to: the e-mail recipients - :type to: str or list of str + :param to: the e-mail recipients as a string or as a list of strings + :type to: str or list :param subject: the subject of the e-mail :type subject: str - :param cc: the e-mails to CC - :type cc: str or list of str + :param cc: the e-mails to CC as a string or as a list of strings + :type cc: str or list :param body: the e-mail body :type body: str @@ -96,7 +96,7 @@ rest of the framework. appear on the e-mail's attachment, ``mimetype`` is the mimetype of the attachment and ``file_object`` is a readable file object with the contents of the attachment - :type attachs: iterable + :type attachs: collections.abc.Iterable :param mimetype: the MIME type of the e-mail :type mimetype: str diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index e5c99e5b1..ef50c9f5c 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -57,7 +57,7 @@ value of one of their fields:: adapter = ItemAdapter(item) year = adapter['year'] if year not in self.year_to_exporter: - f = open('{}.xml'.format(year), 'wb') + f = open(f'{year}.xml', 'wb') exporter = XmlItemExporter(f) exporter.start_exporting() self.year_to_exporter[year] = exporter @@ -98,7 +98,7 @@ Example:: import scrapy def serialize_price(value): - return '$ %s' % str(value) + return f'$ {str(value)}' class Product(scrapy.Item): name = scrapy.Field() @@ -122,7 +122,7 @@ Example:: def serialize_field(self, field, name, value): if field == 'price': - return '$ %s' % str(value) + return f'$ {str(value)}' return super(Product, self).serialize_field(field, name, value) .. _topics-exporters-reference: @@ -166,8 +166,7 @@ BaseItemExporter By default, this method looks for a serializer :ref:`declared in the item field ` and returns the result of applying that serializer to the value. If no serializer is found, it returns the - value unchanged except for ``unicode`` values which are encoded to - ``str`` using the encoding declared in the :attr:`encoding` attribute. + value unchanged. :param field: the field being serialized. If the source :ref:`item object ` does not define field metadata, *field* is an empty @@ -217,10 +216,7 @@ BaseItemExporter .. attribute:: encoding - The encoding that will be used to encode unicode values. This only - affects unicode values (which are always serialized to str using this - encoding). Other value types are passed unchanged to the specific - serialization library. + The output character encoding. .. attribute:: indent @@ -296,7 +292,7 @@ XmlItemExporter CsvItemExporter --------------- -.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs) +.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs) Exports items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the @@ -309,12 +305,17 @@ CsvItemExporter :param include_headers_line: If enabled, makes the exporter output a header line with the field names taken from :attr:`BaseItemExporter.fields_to_export` or the first exported item fields. - :type include_headers_line: boolean + :type include_headers_line: bool :param join_multivalued: The char (or chars) that will be used for joining multi-valued fields, if found. :type include_headers_line: str + :param errors: The optional string that specifies how encoding and decoding + errors are to be handled. For more information see + :class:`io.TextIOWrapper`. + :type errors: str + The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the :func:`csv.writer` function, so you can use any :func:`csv.writer` function diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 0fc83e645..14096ada4 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -288,8 +288,6 @@ If zero (or non set), spiders won't be closed by number of passed items. CLOSESPIDER_PAGECOUNT """"""""""""""""""""" -.. versionadded:: 0.11 - Default: ``0`` An integer which specifies the maximum number of responses to crawl. If the spider @@ -302,8 +300,6 @@ number of crawled responses. CLOSESPIDER_ERRORCOUNT """""""""""""""""""""" -.. versionadded:: 0.11 - Default: ``0`` An integer which specifies the maximum number of errors to receive before diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0dc73c513..9fb2189e8 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -4,8 +4,6 @@ Feed exports ============ -.. versionadded:: 0.10 - One of the most frequently required features when implementing scrapers is being able to store the scraped data properly and, quite often, that means generating an "export file" with the scraped data (commonly called "export @@ -100,6 +98,7 @@ The storages backends supported out of the box are: * :ref:`topics-feed-storage-fs` * :ref:`topics-feed-storage-ftp` * :ref:`topics-feed-storage-s3` (requires botocore_) + * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) * :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are @@ -169,6 +168,9 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +This storage backend uses :ref:`delayed file delivery `. + + .. _topics-feed-storage-s3: S3 @@ -194,11 +196,16 @@ You can also define a custom ACL for exported feeds using this setting: * :setting:`FEED_STORAGE_S3_ACL` +This storage backend uses :ref:`delayed file delivery `. + + .. _topics-feed-storage-gcs: Google Cloud Storage (GCS) -------------------------- +.. versionadded:: 2.3 + The feeds are stored on `Google Cloud Storage`_. * URI scheme: ``gs`` @@ -206,7 +213,7 @@ The feeds are stored on `Google Cloud Storage`_. * ``gs://mybucket/path/to/export.csv`` - * Required external libraries: `google-cloud-storage `_. + * Required external libraries: `google-cloud-storage`_. For more information about authentication, please refer to `Google Cloud documentation `_. @@ -215,6 +222,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following * :setting:`FEED_STORAGE_GCS_ACL` * :setting:`GCS_PROJECT_ID` +This storage backend uses :ref:`delayed file delivery `. + +.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python + + .. _topics-feed-storage-stdout: Standard output @@ -227,6 +239,26 @@ The feeds are written to the standard output of the Scrapy process. * Required external libraries: none +.. _delayed-file-delivery: + +Delayed file delivery +--------------------- + +As indicated above, some of the described storage backends use delayed file +delivery. + +These storage backends do not upload items to the feed URI as those items are +scraped. Instead, Scrapy writes items into a temporary local file, and only +once all the file contents have been written (i.e. at the end of the crawl) is +that file uploaded to the feed URI. + +If you want item delivery to start earlier when using one of these storage +backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items +in multiple files, with the specified maximum item count per file. That way, as +soon as a file reaches the maximum item count, that file is delivered to the +feed URI, allowing item delivery to start way before the end of the crawl. + + Settings ======== @@ -241,6 +273,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` + * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport @@ -256,6 +289,7 @@ Default: ``{}`` A dictionary in which every key is a feed URI (or a :class:`pathlib.Path` object) and each value is a nested dictionary containing configuration parameters for the specific feed. + This setting is required for enabling the feed export feature. See :ref:`topics-feed-storage-backends` for supported URI schemes. @@ -283,15 +317,43 @@ For instance:: } The following is a list of the accepted keys and the setting that is used -as a fallback value if that key is not provided for a specific feed definition. +as a fallback value if that key is not provided for a specific feed definition: + +- ``format``: the :ref:`serialization format `. + + This setting is mandatory, there is no fallback value. + +- ``batch_item_count``: falls back to + :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + +- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`. + +- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. + +- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. + +- ``overwrite``: whether to overwrite the file if it already exists + (``True``) or append to its content (``False``). + + The default value depends on the :ref:`storage backend + `: + + - :ref:`topics-feed-storage-fs`: ``False`` + + - :ref:`topics-feed-storage-ftp`: ``True`` + + .. note:: Some FTP servers may not support appending to files (the + ``APPE`` FTP command). + + - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported + `_) + + - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) + +- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`. + +- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. -* ``format``: the serialization format to be used for the feed. - See :ref:`topics-feed-format` for possible values. - Mandatory, no fallback setting -* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING` -* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` -* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` -* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` .. setting:: FEED_EXPORT_ENCODING @@ -446,6 +508,109 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter 'csv': None, } + +.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT + +FEED_EXPORT_BATCH_ITEM_COUNT +----------------------------- + +Default: ``0`` + +If assigned an integer number higher than ``0``, Scrapy generates multiple output files +storing up to the specified number of items in each output file. + +When generating multiple output files, you must use at least one of the following +placeholders in the feed URI to indicate how the different output file names are +generated: + +* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created + (e.g. ``2020-03-28T14-45-08.237134``) + +* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch. + + Use :ref:`printf-style string formatting ` to + alter the number format. For example, to make the batch ID a 5-digit + number by introducing leading zeroes as needed, use ``%(batch_id)05d`` + (e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``). + +For instance, if your settings include:: + + FEED_EXPORT_BATCH_ITEM_COUNT = 100 + +And your :command:`crawl` command line is:: + + scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json" + +The command line above can generate a directory tree like:: + + ->projectname + -->dirname + --->1-filename2020-03-28T14-45-08.237134.json + --->2-filename2020-03-28T14-45-09.148903.json + --->3-filename2020-03-28T14-45-10.046092.json + +Where the first and second files contain exactly 100 items. The last one contains +100 items or fewer. + + +.. setting:: FEED_URI_PARAMS + +FEED_URI_PARAMS +--------------- + +Default: ``None`` + +A string with the import path of a function to set the parameters to apply with +:ref:`printf-style string formatting ` to the +feed URI. + +The function signature should be as follows: + +.. function:: uri_params(params, spider) + + Return a :class:`dict` of key-value pairs to apply to the feed URI using + :ref:`printf-style string formatting `. + + :param params: default key-value pairs + + Specifically: + + - ``batch_id``: ID of the file batch. See + :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + + If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id`` + is always ``1``. + + - ``batch_time``: UTC date and time, in ISO format with ``:`` + replaced with ``-``. + + See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + + - ``time``: ``batch_time``, with microseconds set to ``0``. + :type params: dict + + :param spider: source spider of the feed items + :type spider: scrapy.spiders.Spider + +For example, to include the :attr:`name ` of the +source spider in the feed URI: + +#. Define the following function somewhere in your project:: + + # myproject/utils.py + def uri_params(params, spider): + return {**params, 'spider_name': spider.name} + +#. Point :setting:`FEED_URI_PARAMS` to that function in your settings:: + + # myproject/settings.py + FEED_URI_PARAMS = 'myproject.utils.uri_params' + +#. Use ``%(spider_name)s`` in your feed URI:: + + scrapy crawl -o "%(spider_name)s.jl" + + .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ .. _botocore: https://github.com/boto/botocore diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index cd6a6d47e..6287ee0ad 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -96,7 +96,7 @@ contain a price:: adapter['price'] = adapter['price'] * self.vat_factor return item else: - raise DropItem("Missing price in %s" % item) + raise DropItem(f"Missing price in {item}") Write items to a JSON file @@ -211,7 +211,7 @@ item. # Save screenshot to file, filename will be hash of url. url = adapter["url"] url_hash = hashlib.md5(url.encode("utf8")).hexdigest() - filename = "{}.png".format(url_hash) + filename = f"{url_hash}.png" with open(filename, "wb") as f: f.write(response.body) @@ -240,7 +240,7 @@ returns multiples items with the same id:: def process_item(self, item, spider): adapter = ItemAdapter(item) if adapter['id'] in self.ids_seen: - raise DropItem("Duplicate item found: %r" % item) + raise DropItem(f"Duplicate item found: {item!r}") else: self.ids_seen.add(adapter['id']) return item diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 3224241fc..b895b95cb 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -102,7 +102,7 @@ A real example Let's see a concrete example of a hypothetical case of memory leaks. Suppose we have some spider with a line similar to this one:: - return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, + return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}", callback=self.parse, cb_kwargs={'referer': response}) That line is passing a response reference inside a request which effectively @@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. :param ignore: if given, all objects from the specified class (or tuple of classes) will be ignored. - :type ignore: class or classes tuple + :type ignore: type or tuple .. function:: get_oldest(class_name) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 0162a331a..ed32411b0 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -46,13 +46,13 @@ LxmlLinkExtractor :param allow: a single regular expression (or list of regular expressions) that the (absolute) urls must match in order to be extracted. If not given (or empty), it will match all links. - :type allow: a regular expression (or list of) + :type allow: str or list :param deny: a single regular expression (or list of regular expressions) that the (absolute) urls must match in order to be excluded (i.e. not extracted). It has precedence over the ``allow`` parameter. If not given (or empty) it won't exclude any links. - :type deny: a regular expression (or list of) + :type deny: str or list :param allow_domains: a single value or a list of string containing domains which will be considered for extracting the links @@ -88,7 +88,7 @@ LxmlLinkExtractor that the link's text must match in order to be extracted. If not given (or empty), it will match all links. If a list of regular expressions is given, the link will be extracted if it matches at least one. - :type restrict_text: a regular expression (or list of) + :type restrict_text: str or list :param tags: a tag or a list of tags to consider when extracting links. Defaults to ``('a', 'area')``. @@ -106,11 +106,11 @@ LxmlLinkExtractor different for requests with canonicalized and raw URLs. If you're using LinkExtractor to follow links it is more robust to keep the default ``canonicalize=False``. - :type canonicalize: boolean + :type canonicalize: bool :param unique: whether duplicate filtering should be applied to extracted links. - :type unique: boolean + :type unique: bool :param process_value: a function which receives each value extracted from the tag and attributes scanned and can modify the value and return a @@ -132,7 +132,7 @@ LxmlLinkExtractor if m: return m.group(1) - :type process_value: callable + :type process_value: collections.abc.Callable :param strip: whether to strip whitespaces from extracted attributes. According to HTML5 standard, leading and trailing whitespaces @@ -141,7 +141,7 @@ LxmlLinkExtractor elements, etc., so LinkExtractor strips space chars by default. Set ``strip=False`` to turn it off (e.g. if you're extracting urls from elements or attributes which allow leading/trailing whitespaces). - :type strip: boolean + :type strip: bool .. automethod:: extract_links diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index d0eeb4097..c0f534493 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -193,10 +193,10 @@ Item Loaders are declared using a class definition syntax. Here is an example:: default_output_processor = TakeFirst() - name_in = MapCompose(unicode.title) + name_in = MapCompose(str.title) name_out = Join() - price_in = MapCompose(unicode.strip) + price_in = MapCompose(str.strip) # ... @@ -237,10 +237,10 @@ metadata. Here is an example:: >>> from scrapy.loader import ItemLoader >>> il = ItemLoader(item=Product()) ->>> il.add_value('name', [u'Welcome to my', u'website']) ->>> il.add_value('price', [u'€', u'1000']) +>>> il.add_value('name', ['Welcome to my', 'website']) +>>> il.add_value('price', ['€', '1000']) >>> il.load_item() -{'name': u'Welcome to my website', 'price': u'1000'} +{'name': 'Welcome to my website', 'price': '1000'} The precedence order, for both input and output processors, is as follows: diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 1f995ce14..487e26b8e 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -412,15 +412,16 @@ See here the methods that you can override in your custom Files Pipeline: .. class:: FilesPipeline - .. method:: file_path(self, request, response=None, info=None) + .. method:: file_path(self, request, response=None, info=None, *, item=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified :class:`response `. In addition to ``response``, this method receives the original - :class:`request ` and - :class:`info `. + :class:`request `, + :class:`info ` and + :class:`item ` You can override this method to customize the download path of each file. @@ -436,9 +437,12 @@ See here the methods that you can override in your custom Files Pipeline: class MyFilesPipeline(FilesPipeline): - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): return 'files/' + os.path.basename(urlparse(request.url).path) + Similarly, you can use the ``item`` to determine the file path based on some item + property. + By default the :meth:`file_path` method returns ``full/.``. @@ -544,15 +548,16 @@ See here the methods that you can override in your custom Images Pipeline: The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`, customizing the field names and adding custom behavior for images. - .. method:: file_path(self, request, response=None, info=None) + .. method:: file_path(self, request, response=None, info=None, *, item=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified :class:`response `. In addition to ``response``, this method receives the original - :class:`request ` and - :class:`info `. + :class:`request `, + :class:`info ` and + :class:`item ` You can override this method to customize the download path of each file. @@ -568,9 +573,12 @@ See here the methods that you can override in your custom Images Pipeline: class MyImagesPipeline(ImagesPipeline): - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): return 'files/' + os.path.basename(urlparse(request.url).path) + Similarly, you can use the ``item`` to determine the file path based on some item + property. + By default the :meth:`file_path` method returns ``full/.``. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index fbd8e4b73..30b1945d0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -33,7 +33,7 @@ Request objects :param url: the URL of this request If the URL is invalid, a :exc:`ValueError` exception is raised. - :type url: string + :type url: str :param callback: the function that will be called with the response of this request (once it's downloaded) as its first parameter. For more information @@ -42,21 +42,21 @@ Request objects :meth:`~scrapy.spiders.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. - :type callback: callable + :type callback: collections.abc.Callable :param method: the HTTP method of this request. Defaults to ``'GET'``. - :type method: string + :type method: str :param meta: the initial values for the :attr:`Request.meta` attribute. If given, the dict passed in this parameter will be shallow copied. :type meta: dict - :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If - ``body`` is not given, an empty string is stored. Regardless of the - type of this argument, the final value stored will be a ``str`` (never - ``unicode`` or ``None``). - :type body: str or unicode + :param body: the request body. If a string is passed, then it's encoded as + bytes using the ``encoding`` passed (which defaults to ``utf-8``). If + ``body`` is not given, an empty bytes object is stored. Regardless of the + type of this argument, the final value stored will be a bytes object + (never a string or ``None``). + :type body: bytes or str :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -106,8 +106,8 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the - body to ``str`` (if given as ``unicode``). - :type encoding: string + body to bytes (if given as a string). + :type encoding: str :param priority: the priority of this request (defaults to ``0``). The priority is used by the scheduler to define the order used to process @@ -119,7 +119,7 @@ Request objects the scheduler. This is used when you want to perform an identical request multiple times, to ignore the duplicates filter. Use it with care, or you will get into crawling loops. Default to ``False``. - :type dont_filter: boolean + :type dont_filter: bool :param errback: a function that will be called if any exception was raised while processing the request. This includes pages that failed @@ -131,7 +131,7 @@ Request objects .. versionchanged:: 2.0 The *callback* parameter is no longer required when the *errback* parameter is specified. - :type errback: callable + :type errback: collections.abc.Callable :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list @@ -159,7 +159,7 @@ Request objects .. attribute:: Request.body - A str that contains the request body. + The request body as bytes. This attribute is read-only. To change the body of a Request use :meth:`replace`. @@ -485,7 +485,7 @@ fields with form data from :class:`Response` objects. :param formdata: is a dictionary (or iterable of (key, value) tuples) containing HTML Form data which will be url-encoded and assigned to the body of the request. - :type formdata: dict or iterable of tuples + :type formdata: dict or collections.abc.Iterable The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: @@ -517,20 +517,20 @@ fields with form data from :class:`Response` objects. :type response: :class:`Response` object :param formname: if given, the form with name attribute set to this value will be used. - :type formname: string + :type formname: str :param formid: if given, the form with id attribute set to this value will be used. - :type formid: string + :type formid: str :param formxpath: if given, the first form that matches the xpath will be used. - :type formxpath: string + :type formxpath: str :param formcss: if given, the first form that matches the css selector will be used. - :type formcss: string + :type formcss: str :param formnumber: the number of form to use, when the response contains multiple forms. The first one (and also the default) is ``0``. - :type formnumber: integer + :type formnumber: int :param formdata: fields to override in the form data. If a field was already present in the response ``
`` element, its value is @@ -548,23 +548,11 @@ fields with form data from :class:`Response` objects. :param dont_click: If True, the form data will be submitted without clicking in any element. - :type dont_click: boolean + :type dont_click: bool The other parameters of this class method are passed directly to the :class:`FormRequest` ``__init__`` method. - .. versionadded:: 0.10.3 - The ``formname`` parameter. - - .. versionadded:: 0.17 - The ``formxpath`` parameter. - - .. versionadded:: 1.1.0 - The ``formcss`` parameter. - - .. versionadded:: 1.1.0 - The ``formid`` parameter. - Request usage examples ---------------------- @@ -636,7 +624,7 @@ dealing with JSON requests. if :attr:`Request.body` argument is provided this parameter will be ignored. if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be set to ``'POST'`` automatically. - :type data: JSON serializable object + :type data: object :param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize data into JSON format. @@ -663,16 +651,16 @@ Response objects downloaded (by the Downloader) and fed to the Spiders for processing. :param url: the URL of this response - :type url: string + :type url: str :param status: the HTTP status of the response. Defaults to ``200``. - :type status: integer + :type status: int :param headers: the headers of this response. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). :type headers: dict - :param body: the response body. To access the decoded text as str you can use + :param body: the response body. To access the decoded text as a string, use ``response.text`` from an encoding-aware :ref:`Response subclass `, such as :class:`TextResponse`. @@ -720,10 +708,10 @@ Response objects .. attribute:: Response.body - The body of this Response. Keep in mind that Response.body - is always a bytes object. If you want the unicode version use - :attr:`TextResponse.text` (only available in :class:`TextResponse` - and subclasses). + The response body as bytes. + + If you want the body as a string, use :attr:`TextResponse.text` (only + available in :class:`TextResponse` and subclasses). This attribute is read-only. To change the body of a Response use :meth:`replace`. @@ -842,18 +830,18 @@ TextResponse objects is the same as for the :class:`Response` class and is not documented here. :param encoding: is a string which contains the encoding to use for this - response. If you create a :class:`TextResponse` object with a unicode - body, it will be encoded using this encoding (remember the body attribute - is always a string). If ``encoding`` is ``None`` (default value), the - encoding will be looked up in the response headers and body instead. - :type encoding: string + response. If you create a :class:`TextResponse` object with a string as + body, it will be converted to bytes encoded using this encoding. If + *encoding* is ``None`` (default), the encoding will be looked up in the + response headers and body instead. + :type encoding: str :class:`TextResponse` objects support the following attributes in addition to the standard :class:`Response` ones: .. attribute:: TextResponse.text - Response body, as unicode. + Response body, as a string. The same as ``response.body.decode(response.encoding)``, but the result is cached after the first call, so you can access @@ -861,9 +849,11 @@ TextResponse objects .. note:: - ``unicode(response.body)`` is not a correct way to convert response - body to unicode: you would be using the system default encoding - (typically ``ascii``) instead of the response encoding. + ``str(response.body)`` is not a correct way to convert the response + body into a string: + + >>> str(b'body') + "b'body'" .. attribute:: TextResponse.encoding diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index bb46ea80f..b576fde91 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``: Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or -markup as an unicode string (in ``text`` argument). +markup as a string (in ``text`` argument). + Usually there is no need to construct Scrapy selectors manually: ``response`` object is available in Spider callbacks, so in most cases it is more convenient to use ``response.css()`` and ``response.xpath()`` @@ -327,8 +328,9 @@ too. Here's an example: 'Name: My image 5
'] >>> for index, link in enumerate(links): -... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get()) -... print('Link number %d points to url %r and image %r' % args) +... href_xpath = link.xpath('@href').get() +... img_xpath = link.xpath('img/@src').get() +... print(f'Link number {index} points to url {href_xpath!r} and image {img_xpath!r}') Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg' Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg' Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg' @@ -383,7 +385,7 @@ Using selectors with regular expressions :class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting data using regular expressions. However, unlike using ``.xpath()`` or -``.css()`` methods, ``.re()`` returns a list of unicode strings. So you +``.css()`` methods, ``.re()`` returns a list of strings. So you can't construct nested ``.re()`` calls. Here's an example used to extract image names from the :ref:`HTML code @@ -734,7 +736,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's Example selecting links in list item with a "class" attribute ending with a digit: >>> from scrapy import Selector ->>> doc = u""" +>>> doc = """ ...
...
    ...
  • first item
    • @@ -765,7 +767,7 @@ extracting text elements for example. Example extracting microdata (sample content taken from https://schema.org/Product) with groups of itemscopes and corresponding itemprops:: - >>> doc = u""" + >>> doc = """ ...
    ... Kenmore White 17" Microwave ... Kenmore 17" Microwave @@ -821,7 +823,7 @@ with groups of itemscopes and corresponding itemprops:: ... props = scope.xpath(''' ... set:difference(./descendant::*/@itemprop, ... .//*[@itemscope]/*/@itemprop)''') - ... print(" properties: %s" % (props.getall())) + ... print(f" properties: {props.getall()}") ... print("") current scope: ['http://schema.org/Product'] @@ -989,7 +991,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this:: sel.xpath("//h1") 2. Extract the text of all ``

    `` elements from an HTML response body, - returning a list of unicode strings:: + returning a list of strings:: sel.xpath("//h1").getall() # this includes the h1 tag sel.xpath("//h1/text()").getall() # this excludes the h1 tag diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ee0cb9690..8480381c9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -98,6 +98,32 @@ class. The global defaults are located in the ``scrapy.settings.default_settings`` module and documented in the :ref:`topics-settings-ref` section. + +Import paths and classes +======================== + +.. versionadded:: VERSION + +When a setting references a callable object to be imported by Scrapy, such as a +class or a function, there are two different ways you can specify that object: + +- As a string containing the import path of that object + +- As the object itself + +For example:: + + from mybot.pipelines.validate import ValidateMyItem + ITEM_PIPELINES = { + # passing the classname... + ValidateMyItem: 300, + # ...equals passing the class path + 'mybot.pipelines.validate.ValidateMyItem': 300, + } + +.. note:: Passing non-callable objects is not supported. + + How to access settings ====================== @@ -110,7 +136,7 @@ In a spider, the settings are available through ``self.settings``:: start_urls = ['http://example.com'] def parse(self, response): - print("Existing settings: %s" % self.settings.attributes.keys()) + print(f"Existing settings: {self.settings.attributes.keys()}") .. note:: The ``settings`` attribute is set in the base Spider class after the spider @@ -216,6 +242,26 @@ Default: ``None`` The name of the region associated with the AWS client. +.. setting:: ASYNCIO_EVENT_LOOP + +ASYNCIO_EVENT_LOOP +------------------ + +Default: ``None`` + +Import path of a given asyncio event loop class. + +If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the +asyncio event loop to be used with it. Set the setting to the import path of the +desired asyncio event loop class. If the setting is set to ``None`` the default asyncio +event loop will be used. + +If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor` +function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop +class to be used. + +Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`. + .. setting:: BOT_NAME BOT_NAME @@ -1030,8 +1076,6 @@ See :ref:`topics-extensions-ref-memusage`. MEMUSAGE_CHECK_INTERVAL_SECONDS ------------------------------- -.. versionadded:: 1.1 - Default: ``60.0`` Scope: ``scrapy.extensions.memusage`` @@ -1336,8 +1380,6 @@ as a name. SPIDER_LOADER_WARN_ONLY ----------------------- -.. versionadded:: 1.3.3 - Default: ``False`` By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 255ba9d3f..1d99d8c28 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -423,6 +423,11 @@ response_received :param spider: the spider for which the response is intended :type spider: :class:`~scrapy.spiders.Spider` object +.. note:: The ``request`` argument might not contain the original request that + reached the downloader, if a :ref:`topics-downloader-middleware` modifies + the :class:`~scrapy.http.Response` object and sets a specific ``request`` + attribute. + response_downloaded ~~~~~~~~~~~~~~~~~~~ diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c6cbdba76..fc114a63f 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -146,8 +146,6 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_start_requests(start_requests, spider) - .. versionadded:: 0.15 - This method is called with the start requests of the spider, and works similarly to the :meth:`process_spider_output` method, except that it doesn't have a response associated and must return only requests (not @@ -341,8 +339,6 @@ RefererMiddleware settings REFERER_ENABLED ^^^^^^^^^^^^^^^ -.. versionadded:: 0.15 - Default: ``True`` Whether to enable referer middleware. @@ -352,8 +348,6 @@ Whether to enable referer middleware. REFERRER_POLICY ^^^^^^^^^^^^^^^ -.. versionadded:: 1.4 - Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` .. reqmeta:: referrer_policy diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 3ed1ada99..c640e1d11 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -282,7 +282,7 @@ Spiders can access arguments in their `__init__` methods:: def __init__(self, category=None, *args, **kwargs): super(MySpider, self).__init__(*args, **kwargs) - self.start_urls = ['http://www.example.com/categories/%s' % category] + self.start_urls = [f'http://www.example.com/categories/{category}'] # ... The default `__init__` method will take any spider arguments @@ -295,7 +295,7 @@ The above example can also be written as follows:: name = 'myspider' def start_requests(self): - yield scrapy.Request('http://www.example.com/categories/%s' % self.category) + yield scrapy.Request(f'http://www.example.com/categories/{self.category}') Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 9acfc3b23..95a3f17d5 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -23,7 +23,7 @@ def main(): _contents = None # A regex that matches standard linkcheck output lines - line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') + line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') # Read lines from the linkcheck output file try: diff --git a/docs/versioning.rst b/docs/versioning.rst index 227085f02..57643ea9a 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -1,7 +1,7 @@ .. _versioning: ============================ -Versioning and API Stability +Versioning and API stability ============================ Versioning @@ -34,7 +34,7 @@ For example: production) -API Stability +API stability ============= API stability was one of the major goals for the *1.0* release. @@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the same way. +.. _deprecation-policy: + +Deprecation policy +================== + +We aim to maintain support for deprecated Scrapy features for at least 1 year. + +For example, if a feature is deprecated in a Scrapy version released on +June 15th 2020, that feature should continue to work in versions released on +June 14th 2021 or before that. + +Any new Scrapy release after a year *may* remove support for that deprecated +feature. + +All deprecated features removed in a Scrapy release are explicitly mentioned in +the :ref:`release notes `. + + .. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index da7a0022b..a6472b1ba 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -37,7 +37,7 @@ class Root(Resource): if now - self.lastmark >= 3: self.lastmark = now qps = len(self.tail) / sum(self.tail) - print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps)) + print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}') if 'latency' in request.args: latency = float(request.args['latency'][0]) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 7554f7eec..f9fb70342 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -27,7 +27,7 @@ class QPSSpider(Spider): slots = 1 def __init__(self, *a, **kw): - super(QPSSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) if self.qps is not None: self.qps = float(self.qps) self.download_delay = 1 / self.qps @@ -37,11 +37,11 @@ class QPSSpider(Spider): def start_requests(self): url = self.benchurl if self.latency is not None: - url += '?latency={0}'.format(self.latency) + url += f'?latency={self.latency}' slots = int(self.slots) if slots > 1: - urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)] + urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)] else: urls = [url] diff --git a/pylintrc b/pylintrc index 129c7bf7d..5b6b9fab0 100644 --- a/pylintrc +++ b/pylintrc @@ -68,6 +68,7 @@ disable=abstract-method, pointless-statement, pointless-string-statement, protected-access, + raise-missing-from, redefined-argument-from-local, redefined-builtin, redefined-outer-name, @@ -75,6 +76,7 @@ disable=abstract-method, signature-differs, singleton-comparison, super-init-not-called, + super-with-arguments, superfluous-parens, too-few-public-methods, too-many-ancestors, diff --git a/pytest.ini b/pytest.ini index 663c5cc78..ca8191f42 100644 --- a/pytest.ini +++ b/pytest.ini @@ -40,3 +40,4 @@ flake8-ignore = scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 tests/test_loader.py E741 + diff --git a/scrapy/VERSION b/scrapy/VERSION index ccbccc3dc..276cbf9e2 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.2.0 +2.3.0 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index f0259a9b7..4326ca4aa 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 5, 2): - print("Scrapy %s requires Python 3.5.2" % __version__) +if sys.version_info < (3, 6): + print("Scrapy %s requires Python 3.6+" % __version__) sys.exit(1) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 306ddfc9e..be3299063 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -42,7 +42,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'): if inspect.isclass(obj): cmds[entry_point.name] = obj() else: - raise Exception("Invalid entry point %s" % entry_point.name) + raise Exception(f"Invalid entry point {entry_point.name}") return cmds @@ -65,11 +65,11 @@ def _pop_command_name(argv): def _print_header(settings, inproject): + version = scrapy.__version__ if inproject: - print("Scrapy %s - project: %s\n" % (scrapy.__version__, - settings['BOT_NAME'])) + print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n") else: - print("Scrapy %s - no active project\n" % scrapy.__version__) + print(f"Scrapy {version} - no active project\n") def _print_commands(settings, inproject): @@ -79,7 +79,7 @@ def _print_commands(settings, inproject): print("Available commands:") cmds = _get_commands_dict(settings, inproject) for cmdname, cmdclass in sorted(cmds.items()): - print(" %-13s %s" % (cmdname, cmdclass.short_desc())) + print(f" {cmdname:<13} {cmdclass.short_desc()}") if not inproject: print() print(" [ more ] More commands available when run from project directory") @@ -89,7 +89,7 @@ def _print_commands(settings, inproject): def _print_unknown_command(settings, cmdname, inproject): _print_header(settings, inproject) - print("Unknown command: %s\n" % cmdname) + print(f"Unknown command: {cmdname}\n") print('Use "scrapy" to see available commands') @@ -131,7 +131,7 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax()) + parser.usage = f"scrapy {cmdname} {cmd.syntax()}" parser.description = cmd.long_desc() settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings @@ -153,7 +153,7 @@ def _run_command(cmd, args, opts): def _run_command_profiled(cmd, args, opts): if opts.profile: - sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile) + sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() p = cProfile.Profile() p.runctx('cmd.run(args, opts)', globals(), loc) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 57ce4e522..23ccffcd9 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -61,7 +61,7 @@ class ScrapyCommand: group.add_option("--logfile", metavar="FILE", help="log file. if omitted stderr will be used") group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help="log level (default: %s)" % self.settings['LOG_LEVEL']) + help=f"log level (default: {self.settings['LOG_LEVEL']})") group.add_option("--nolog", action="store_true", help="disable logging completely") group.add_option("--profile", metavar="FILE", default=None, @@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand): parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") + help="append scraped items to the end of FILE (use - for stdout)") + parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append", + help="dump scraped items into FILE, overwriting any existing file") parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") + help="format to use for dumping items") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -125,6 +127,11 @@ class BaseRunSpiderCommand(ScrapyCommand): opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) + if opts.output or opts.overwrite_output: + feeds = feed_process_params_from_cli( + self.settings, + opts.output, + opts.output_format, + opts.overwrite_output, + ) self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index c9f3b38e0..999c987ea 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider): def start_requests(self): qargs = {'total': self.total, 'show': self.show} - url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1)) + url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}' return [scrapy.Request(url, dont_filter=True)] def parse(self, response): diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 9d4437a47..ae21d86e6 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult): plural = "s" if run != 1 else "" writeln(self.separator2) - writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start)) + writeln(f"Ran {run} contract{plural} in {stop - start:.3f}s") writeln() infos = [] @@ -25,14 +25,14 @@ class TextTestResult(_TextTestResult): write("FAILED") failed, errored = map(len, (self.failures, self.errors)) if failed: - infos.append("failures=%d" % failed) + infos.append(f"failures={failed}") if errored: - infos.append("errors=%d" % errored) + infos.append(f"errors={errored}") else: write("OK") if infos: - writeln(" (%s)" % (", ".join(infos),)) + writeln(f" ({', '.join(infos)})") else: write("\n") @@ -78,19 +78,19 @@ class Command(ScrapyCommand): elif tested_methods: self.crawler_process.crawl(spidercls) - # start checks - if opts.list: - for spider, methods in sorted(contract_reqs.items()): - if not methods and not opts.verbose: - continue - print(spider) - for method in sorted(methods): - print(' * %s' % method) - else: - start = time.time() - self.crawler_process.start() - stop = time.time() + # start checks + if opts.list: + for spider, methods in sorted(contract_reqs.items()): + if not methods and not opts.verbose: + continue + print(spider) + for method in sorted(methods): + print(f' * {method}') + else: + start = time.time() + self.crawler_process.start() + stop = time.time() - result.printErrors() - result.printSummary(start, stop) - self.exitcode = int(not result.wasSuccessful()) + result.printErrors() + result.printSummary(start, stop) + self.exitcode = int(not result.wasSuccessful()) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 25d843a53..177b20143 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -32,8 +32,8 @@ class Command(ScrapyCommand): try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: - return self._err("Spider not found: %s" % args[0]) + return self._err(f"Spider not found: {args[0]}") sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace('.pyc', '.py') - self.exitcode = os.system('%s "%s"' % (editor, sfile)) + self.exitcode = os.system(f'{editor} "{sfile}"') diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 4c7548e9c..72248bded 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -66,31 +66,25 @@ class Command(ScrapyCommand): print("Cannot create a spider with the same name as your project") return - try: - spidercls = self.crawler_process.spider_loader.load(name) - except KeyError: - pass - else: - # if spider already exists and not --force then halt - if not opts.force: - print("Spider %r already exists in module:" % name) - print(" %s" % spidercls.__module__) - return + if not opts.force and self._spider_exists(name): + return + template_file = self._find_template(opts.template) if template_file: self._genspider(module, name, domain, opts.template, template_file) if opts.edit: - self.exitcode = os.system('scrapy edit "%s"' % name) + self.exitcode = os.system(f'scrapy edit "{name}"') def _genspider(self, module, name, domain, template_name, template_file): """Generate the spider module, based on the given template""" + capitalized_module = ''.join(s.capitalize() for s in module.split('_')) tvars = { 'project_name': self.settings.get('BOT_NAME'), 'ProjectName': string_camelcase(self.settings.get('BOT_NAME')), 'module': module, 'name': name, 'domain': domain, - 'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_')) + 'classname': f'{capitalized_module}Spider' } if self.settings.get('NEWSPIDER_MODULE'): spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) @@ -98,26 +92,54 @@ class Command(ScrapyCommand): else: spiders_module = None spiders_dir = "." - spider_file = "%s.py" % join(spiders_dir, module) + spider_file = f"{join(spiders_dir, module)}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print("Created spider %r using template %r " - % (name, template_name), end=('' if spiders_module else '\n')) + print(f"Created spider {name!r} using template {template_name!r} ", + end=('' if spiders_module else '\n')) if spiders_module: - print("in module:\n %s.%s" % (spiders_module.__name__, module)) + print("in module:\n {spiders_module.__name__}.{module}") def _find_template(self, template): - template_file = join(self.templates_dir, '%s.tmpl' % template) + template_file = join(self.templates_dir, f'{template}.tmpl') if exists(template_file): return template_file - print("Unable to find template: %s\n" % template) + print(f"Unable to find template: {template}\n") print('Use "scrapy genspider --list" to see all available templates.') def _list_templates(self): print("Available templates:") for filename in sorted(os.listdir(self.templates_dir)): if filename.endswith('.tmpl'): - print(" %s" % splitext(filename)[0]) + print(f" {splitext(filename)[0]}") + + def _spider_exists(self, name): + if not self.settings.get('NEWSPIDER_MODULE'): + # if run as a standalone command and file with same filename already exists + if exists(name + ".py"): + print(f"{abspath(name + '.py')} already exists") + return True + return False + + try: + spidercls = self.crawler_process.spider_loader.load(name) + except KeyError: + pass + else: + # if spider with same name exists + print(f"Spider {name!r} already exists in module:") + print(f" {spidercls.__module__}") + return True + + # a file with the same name exists in the target directory + spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) + spiders_dir = dirname(spiders_module.__file__) + spiders_dir_abs = abspath(spiders_dir) + if exists(join(spiders_dir_abs, name + ".py")): + print(f"{join(spiders_dir_abs, (name + '.py'))} already exists") + return True + + return False @property def templates_dir(self): diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index abc8ba9ff..83ee074da 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -96,13 +96,13 @@ class Command(BaseRunSpiderCommand): if opts.verbose: for level in range(1, self.max_level + 1): - print('\n>>> DEPTH LEVEL: %s <<<' % level) + print(f'\n>>> DEPTH LEVEL: {level} <<<') if not opts.noitems: self.print_items(level, colour) if not opts.nolinks: self.print_requests(level, colour) else: - print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level) + print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<') if not opts.noitems: self.print_items(colour=colour) if not opts.nolinks: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 8c8128d83..91205af5e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -12,7 +12,7 @@ def _import_file(filepath): dirname, file = os.path.split(abspath) fname, fext = os.path.splitext(file) if fext != '.py': - raise ValueError("Not a Python source file: %s" % abspath) + raise ValueError(f"Not a Python source file: {abspath}") if dirname: sys.path = [dirname] + sys.path try: @@ -42,16 +42,16 @@ class Command(BaseRunSpiderCommand): raise UsageError() filename = args[0] if not os.path.exists(filename): - raise UsageError("File not found: %s\n" % filename) + raise UsageError(f"File not found: {filename}\n") try: module = _import_file(filename) except (ImportError, ValueError) as e: - raise UsageError("Unable to load %r: %s\n" % (filename, e)) + raise UsageError(f"Unable to load {filename!r}: {e}\n") require_name = self.settings.getbool('SPIDER_LOADER_REQUIRE_NAME') spclasses = list(iter_spider_classes(module, require_name=require_name)) if not spclasses: - raise UsageError("No spider found in file: %s\n" % filename) + raise UsageError(f"No spider found in file: {filename}\n") spidercls = spclasses.pop() self.crawler_process.crawl(spidercls, **opts.spargs) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e5158d993..1d73fa0cb 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -52,7 +52,7 @@ class Command(ScrapyCommand): print('Error: Project names must begin with a letter and contain' ' only\nletters, numbers and underscores') elif _module_exists(project_name): - print('Error: Module %r already exists' % project_name) + print(f'Error: Module {project_name!r} already exists') else: return True return False @@ -100,7 +100,7 @@ class Command(ScrapyCommand): if exists(join(project_dir, 'scrapy.cfg')): self.exitcode = 1 - print('Error: scrapy.cfg already exists in %s' % abspath(project_dir)) + print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}') return if not self._is_valid_name(project_name): @@ -113,11 +113,11 @@ class Command(ScrapyCommand): path = join(*paths) tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) - print("New Scrapy project '%s', using template directory '%s', " - "created in:" % (project_name, self.templates_dir)) - print(" %s\n" % abspath(project_dir)) + print(f"New Scrapy project '{project_name}', using template directory " + f"'{self.templates_dir}', created in:") + print(f" {abspath(project_dir)}\n") print("You can start your first spider with:") - print(" cd %s" % project_dir) + print(f" cd {project_dir}") print(" scrapy genspider example example.com") @property diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index d0ea72a67..1237610cb 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -23,8 +23,7 @@ class Command(ScrapyCommand): if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) - patt = "%-{}s : %s".format(width) for name, version in versions: - print(patt % (name, version)) + print(f"{name:<{width}} : {version}") else: - print("Scrapy %s" % scrapy.__version__) + print(f"Scrapy {scrapy.__version__}") diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 908bee966..c8f873334 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -11,7 +11,7 @@ class Command(fetch.Command): return "Fetch a URL using the Scrapy downloader and show its contents in a browser" def add_options(self, parser): - super(Command, self).add_options(parser) + super().add_options(parser) parser.remove_option("--headers") def _print_response(self, response, opts): diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 5af3831a2..db0a56e56 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -112,8 +112,8 @@ class Contract: request_cls = None def __init__(self, method, *args): - self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name) - self.testcase_post = _create_testcase(method, '@%s post-hook' % self.name) + self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') + self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') self.args = args def add_pre_hook(self, request, results): @@ -172,8 +172,8 @@ def _create_testcase(method, desc): class ContractTestCase(TestCase): def __str__(_self): - return "[%s] %s (%s)" % (spider, method.__name__, desc) + return f"[{spider}] {method.__name__} ({desc})" - name = '%s_%s' % (spider, method.__name__) + name = f'{spider}_{method.__name__}' setattr(ContractTestCase, name, lambda x: x) return ContractTestCase(name) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 34f0d36d4..9704f5253 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -56,12 +56,11 @@ class ReturnsContract(Contract): } def __init__(self, *args, **kwargs): - super(ReturnsContract, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if len(self.args) not in [1, 2, 3]: raise ValueError( - "Incorrect argument quantity: expected 1, 2 or 3, got %i" - % len(self.args) + f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}" ) self.obj_name = self.args[0] or None self.obj_type_verifier = self.object_type_verifiers[self.obj_name] @@ -88,10 +87,9 @@ class ReturnsContract(Contract): if self.min_bound == self.max_bound: expected = self.min_bound else: - expected = '%s..%s' % (self.min_bound, self.max_bound) + expected = f'{self.min_bound}..{self.max_bound}' - raise ContractFail("Returned %s %s, expected %s" % - (occurrences, self.obj_name, expected)) + raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}") class ScrapesContract(Contract): @@ -106,5 +104,5 @@ class ScrapesContract(Contract): if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] if missing: - missing_str = ", ".join(missing) - raise ContractFail("Missing fields: %s" % missing_str) + missing_fields = ", ".join(missing) + raise ContractFail(f"Missing fields: {missing_fields}") diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index dc5cf1ab8..4f7ab594f 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -41,17 +41,17 @@ class Slot: def __repr__(self): cls_name = self.__class__.__name__ - return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % ( - cls_name, self.concurrency, self.delay, self.randomize_delay) + return (f"{cls_name}(concurrency={self.concurrency!r}, " + f"delay={self.delay:.2f}, " + f"randomize_delay={self.randomize_delay!r})") def __str__(self): return ( - "" % ( - self.concurrency, self.delay, self.randomize_delay, - len(self.active), len(self.queue), len(self.transferring), - datetime.fromtimestamp(self.lastseen).isoformat() - ) + f"" ) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 452242d47..8a7d656a1 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): """ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): - super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._ssl_method = method self.tls_verbose_logging = tls_verbose_logging if tls_ciphers: @@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133) # # * getattr() for `_ssl_method` attribute for context factories - # not calling super(..., self).__init__ + # not calling super().__init__ return CertificateOptions( verify=False, method=getattr(self, 'method', getattr(self, '_ssl_method', None)), diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index e86680978..73aeb2352 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -71,8 +71,7 @@ class DownloadHandlers: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: - raise NotSupported("Unsupported URL scheme '%s': %s" % - (scheme, self._notconfigured[scheme])) + raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}") return handler.download_request(request, spider) @defer.inlineCallbacks diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..1b041c8a8 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -60,11 +60,11 @@ class HTTP11DownloadHandler: settings=settings, crawler=crawler, ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + msg = f""" + '{settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]}' does not accept `method` \ + argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD) and/or \ + `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') @@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf - super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) + super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) self._tunnelReadyDeferred = defer.Deferred() self._tunneledHost = host self._tunneledPort = port @@ -169,8 +169,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): else: extra = rcvd_bytes[:32] self._tunnelReadyDeferred.errback( - TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % ( - self._host, self._port, extra))) + TunnelError('Could not open CONNECT tunnel with proxy ' + f'{self._host}:{self._port} [{extra!r}]') + ) def connectFailed(self, reason): """Propagates the errback to the appropriate deferred.""" @@ -178,7 +179,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def connect(self, protocolFactory): self._protocolFactory = protocolFactory - connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory) + connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) connectDeferred.addErrback(self.connectFailed) return self._tunnelReadyDeferred @@ -215,7 +216,7 @@ class TunnelingAgent(Agent): def __init__(self, reactor, proxyConf, contextFactory=None, connectTimeout=None, bindAddress=None, pool=None): - super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) + super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) self._proxyConf = proxyConf self._contextFactory = contextFactory @@ -235,7 +236,7 @@ class TunnelingAgent(Agent): # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy key = key + self._proxyConf - return super(TunnelingAgent, self)._requestWithEndpoint( + return super()._requestWithEndpoint( key=key, endpoint=endpoint, method=method, @@ -249,7 +250,7 @@ class TunnelingAgent(Agent): class ScrapyProxyAgent(Agent): def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None): - super(ScrapyProxyAgent, self).__init__( + super().__init__( reactor=reactor, connectTimeout=connectTimeout, bindAddress=bindAddress, @@ -371,7 +372,7 @@ class ScrapyAgent: if self._txresponse: self._txresponse._transport.stopProducing() - raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout)) + raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") def _cb_latency(self, result, request, start_time): request.meta['download_latency'] = time() - start_time @@ -394,13 +395,14 @@ class ScrapyAgent: fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) if maxsize and expected_size > maxsize: - error_msg = ("Cancelling download of %(url)s: expected response " - "size (%(size)s) larger than download max size (%(maxsize)s).") - error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} + warning_msg = ("Cancelling download of %(url)s: expected response " + "size (%(size)s) larger than download max size (%(maxsize)s).") + warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} + + logger.warning(warning_msg, warning_args) - logger.error(error_msg, error_args) txresponse._transport._producer.loseConnection() - raise defer.CancelledError(error_msg % error_args) + raise defer.CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " @@ -523,11 +525,11 @@ class _ResponseReader(protocol.Protocol): self._finish_response(flags=["download_stopped"], failure=failure) if self._maxsize and self._bytes_received > self._maxsize: - logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s) in request %(request)s.", - {'bytes': self._bytes_received, - 'maxsize': self._maxsize, - 'request': self._request}) + logger.warning("Received (%(bytes)s) bytes larger than download " + "max size (%(maxsize)s) in request %(request)s.", + {'bytes': self._bytes_received, + 'maxsize': self._maxsize, + 'request': self._request}) # Clear buffer earlier to avoid keeping data in memory for a long time. self._bodybuf.truncate(0) self._finished.cancel() diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 8f63ad974..0ef977893 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -56,7 +56,7 @@ class S3DownloadHandler: import botocore.credentials kw.pop('anon', None) if kw: - raise TypeError('Unexpected keyword arguments: %s' % kw) + raise TypeError(f'Unexpected keyword arguments: {kw}') if not self.anon: SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] self._signer = SignerCls(botocore.credentials.Credentials( @@ -85,14 +85,14 @@ class S3DownloadHandler: scheme = 'https' if request.meta.get('is_secure') else 'http' bucket = p.hostname path = p.path + '?' + p.query if p.query else p.path - url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path) + url = f'{scheme}://{bucket}.s3.amazonaws.com{path}' if self.anon: request = request.replace(url=url) elif self._signer is not None: import botocore.awsrequest awsrequest = botocore.awsrequest.AWSRequest( method=request.method, - url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path), + url=f'{scheme}://s3.amazonaws.com/{bucket}{path}', headers=request.headers.to_unicode_dict(), data=request.body) self._signer.add_auth(awsrequest) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 4c2eea522..b0e612e43 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -36,8 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_request must return None, Response or Request, got %s" - % (method.__self__.__class__.__name__, response.__class__.__name__) + f"Middleware {method.__self__.__class__.__name__}" + ".process_request must return None, Response or " + f"Request, got {response.__class__.__name__}" ) if response: return response @@ -54,8 +55,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_response must return Response or Request, got %s" - % (method.__self__.__class__.__name__, type(response)) + f"Middleware {method.__self__.__class__.__name__}" + ".process_response must return Response or Request, " + f"got {type(response)}" ) if isinstance(response, Request): return response @@ -68,8 +70,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_exception must return None, Response or Request, got %s" - % (method.__self__.__class__.__name__, type(response)) + f"Middleware {method.__self__.__class__.__name__}" + ".process_exception must return None, Response or " + f"Request, got {type(response)}" ) if response: return response diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index e43a3c83e..d9f3750d5 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions): """ def __init__(self, hostname, ctx, verbose_logging=False): - super(ScrapyClientTLSOptions, self).__init__(hostname, ctx) + super().__init__(hostname, ctx) self.verbose_logging = verbose_logging def _identityVerifyingInfoCallback(self, connection, where, ret): diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 355045d74..c13683393 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,9 +1,9 @@ from time import time from urllib.parse import urlparse, urlunparse, urldefrag -from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient -from twisted.internet import defer +from twisted.internet import defer, reactor +from twisted.internet.protocol import ClientFactory from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached @@ -88,22 +88,38 @@ class ScrapyHTTPPageGetter(HTTPClient): self.transport.stopProducing() self.factory.noPage( - defer.TimeoutError("Getting %s took longer than %s seconds." - % (self.factory.url, self.factory.timeout))) + defer.TimeoutError(f"Getting {self.factory.url} took longer " + f"than {self.factory.timeout} seconds.")) -class ScrapyHTTPClientFactory(HTTPClientFactory): - """Scrapy implementation of the HTTPClientFactory overwriting the - setUrl method to make use of our Url object that cache the parse - result. - """ +# This class used to inherit from Twisted’s +# twisted.web.client.HTTPClientFactory. When that class was deprecated in +# Twisted (https://github.com/twisted/twisted/pull/643), we merged its +# non-overriden code into this class. +class ScrapyHTTPClientFactory(ClientFactory): protocol = ScrapyHTTPPageGetter + waiting = 1 noisy = False followRedirect = False afterFoundGet = False + def _build_response(self, body, request): + request.meta['download_latency'] = self.headers_time - self.start_time + status = int(self.status) + headers = Headers(self.response_headers) + respcls = responsetypes.from_args(headers=headers, url=self._url) + return respcls(url=self._url, status=status, headers=headers, body=body) + + def _set_connection_attributes(self, request): + parsed = urlparse_cached(request) + self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) + proxy = request.meta.get('proxy') + if proxy: + self.scheme, _, self.host, self.port, _ = _parse(proxy) + self.path = self.url + def __init__(self, request, timeout=180): self._url = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface @@ -138,21 +154,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): elif self.method == b'POST': self.headers['Content-Length'] = 0 - def _build_response(self, body, request): - request.meta['download_latency'] = self.headers_time - self.start_time - status = int(self.status) - headers = Headers(self.response_headers) - respcls = responsetypes.from_args(headers=headers, url=self._url) - return respcls(url=self._url, status=status, headers=headers, body=body) + def __repr__(self): + return f"<{self.__class__.__name__}: {self.url}>" - def _set_connection_attributes(self, request): - parsed = urlparse_cached(request) - self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) - proxy = request.meta.get('proxy') - if proxy: - self.scheme, _, self.host, self.port, _ = _parse(proxy) - self.path = self.url + def _cancelTimeout(self, result, timeoutCall): + if timeoutCall.active(): + timeoutCall.cancel() + return result + + def buildProtocol(self, addr): + p = ClientFactory.buildProtocol(self, addr) + p.followRedirect = self.followRedirect + p.afterFoundGet = self.afterFoundGet + if self.timeout: + timeoutCall = reactor.callLater(self.timeout, p.timeout) + self.deferred.addBoth(self._cancelTimeout, timeoutCall) + return p def gotHeaders(self, headers): self.headers_time = time() self.response_headers = headers + + def gotStatus(self, version, status, message): + """ + Set the status of the request on us. + @param version: The HTTP version. + @type version: L{bytes} + @param status: The HTTP status code, an integer represented as a + bytestring. + @type status: L{bytes} + @param message: The HTTP status message. + @type message: L{bytes} + """ + self.version, self.status, self.message = version, status, message + + def page(self, page): + if self.waiting: + self.waiting = 0 + self.deferred.callback(page) + + def noPage(self, reason): + if self.waiting: + self.waiting = 0 + self.deferred.errback(reason) + + def clientConnectionFailed(self, _, reason): + """ + When a connection attempt fails, the request cannot be issued. If no + result has yet been provided to the result Deferred, provide the + connection failure reason as an error result. + """ + if self.waiting: + self.waiting = 0 + # If the connection attempt failed, there is nothing more to + # disconnect, so just fire that Deferred now. + self._disconnectedDeferred.callback(None) + self.deferred.errback(reason) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 86a6abb23..93bcdb49a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -171,8 +171,8 @@ class ExecutionEngine: def _handle_downloader_output(self, response, request, spider): if not isinstance(response, (Request, Response, Failure)): raise TypeError( - "Incorrect type: expected Request, Response or Failure, got %s: %r" - % (type(response), response) + "Incorrect type: expected Request, Response or Failure, got " + f"{type(response)}: {response!r}" ) # downloader middleware can return requests (for example, redirects) if isinstance(response, Request): @@ -214,7 +214,7 @@ class ExecutionEngine: def crawl(self, request, spider): if spider not in self.open_spiders: - raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request)) + raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}") self.schedule(request, spider) self.slot.nextcall.schedule() @@ -239,16 +239,21 @@ class ExecutionEngine: def _on_success(response): if not isinstance(response, (Response, Request)): raise TypeError( - "Incorrect type: expected Response or Request, got %s: %r" - % (type(response), response) + "Incorrect type: expected Response or Request, got " + f"{type(response)}: {response!r}" ) if isinstance(response, Response): - response.request = request # tie request to response received - logkws = self.logformatter.crawled(request, response, spider) + if response.request is None: + response.request = request + logkws = self.logformatter.crawled(response.request, response, spider) if logkws is not None: logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) - self.signals.send_catch_log(signals.response_received, - response=response, request=request, spider=spider) + self.signals.send_catch_log( + signal=signals.response_received, + response=response, + request=response.request, + spider=spider, + ) return response def _on_complete(_): @@ -263,7 +268,7 @@ class ExecutionEngine: @defer.inlineCallbacks def open_spider(self, spider, start_requests=(), close_if_idle=True): if not self.has_capacity(): - raise RuntimeError("No free spider slot when opening %r" % spider.name) + raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={'spider': spider}) nextcall = CallLaterOnce(self._next_request, spider) scheduler = self.scheduler_cls.from_crawler(self.crawler) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1ef0790a9..0d3e3450f 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -12,7 +12,7 @@ from scrapy import signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response -from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel +from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output @@ -120,40 +120,40 @@ class Scraper: response, request, deferred = slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, response, request, spider): - """Handle the downloaded response or failure through the spider - callback/errback""" - if not isinstance(response, (Response, Failure)): - raise TypeError( - "Incorrect type: expected Response or Failure, got %s: %r" - % (type(response), response) - ) - - dfd = self._scrape2(response, request, spider) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, response, spider) - dfd.addCallback(self.handle_spider_output, request, response, spider) + def _scrape(self, result, request, spider): + """ + Handle the downloaded response or failure through the spider callback/errback + """ + if not isinstance(result, (Response, Failure)): + raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}") + dfd = self._scrape2(result, request, spider) # returns spider's processed output + dfd.addErrback(self.handle_spider_error, request, result, spider) + dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, request_result, request, spider): - """Handle the different cases of request's result been a Response or a - Failure""" - if not isinstance(request_result, Failure): - return self.spidermw.scrape_response( - self.call_spider, request_result, request, spider) - else: - dfd = self.call_spider(request_result, request, spider) - return dfd.addErrback( - self._log_download_errors, request_result, request, spider) + def _scrape2(self, result, request, spider): + """ + Handle the different cases of request's result been a Response or a Failure + """ + if isinstance(result, Response): + return self.spidermw.scrape_response(self.call_spider, result, request, spider) + else: # result is a Failure + dfd = self.call_spider(result, request, spider) + return dfd.addErrback(self._log_download_errors, result, request, spider) def call_spider(self, result, request, spider): - result.request = request - dfd = defer_result(result) - callback = request.callback or spider._parse - warn_on_generator_with_return_value(spider, callback) - warn_on_generator_with_return_value(spider, request.errback) - dfd.addCallbacks(callback=callback, - errback=request.errback, - callbackKeywords=request.cb_kwargs) + if isinstance(result, Response): + if getattr(result, "request", None) is None: + result.request = request + callback = result.request.callback or spider._parse + warn_on_generator_with_return_value(spider, callback) + dfd = defer_succeed(result) + dfd.addCallback(callback, **result.request.cb_kwargs) + else: # result is a Failure + result.request = request + warn_on_generator_with_return_value(spider, request.errback) + dfd = defer_fail(result) + dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) def handle_spider_error(self, _failure, request, response, spider): @@ -173,7 +173,7 @@ class Scraper: spider=spider ) self.crawler.stats.inc_value( - "spider_exceptions/%s" % _failure.value.__class__.__name__, + f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider ) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 35264a92b..763e0cdf6 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -19,10 +19,7 @@ def _isiterable(possible_iterator): def _fname(f): - return "{}.{}".format( - f.__self__.__class__.__name__, - f.__func__.__name__ - ) + return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" class SpiderMiddlewareManager(MiddlewareManager): @@ -34,7 +31,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) def _add_middleware(self, mw): - super(SpiderMiddlewareManager, self)._add_middleware(mw) + super()._add_middleware(mw) if hasattr(mw, 'process_spider_input'): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_start_requests'): @@ -51,8 +48,9 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = "Middleware {} must return None or raise an exception, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return None " + f"or raise an exception, got {type(result)}") + raise _InvalidOutput(msg) except _InvalidOutput: raise except Exception: @@ -86,8 +84,9 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = "Middleware {} must return None or an iterable, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return None " + f"or an iterable, got {type(result)}") + raise _InvalidOutput(msg) return _failure def process_spider_output(result, start_index=0): @@ -110,8 +109,9 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = _evaluate_iterable(result, method_index + 1, recovered) else: - msg = "Middleware {} must return an iterable, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return an " + f"iterable, got {type(result)}") + raise _InvalidOutput(msg) return MutableChain(result, recovered) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 6f43771e2..4c6b0e496 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner): """ def __init__(self, settings=None, install_root_handler=True): - super(CrawlerProcess, self).__init__(settings) + super().__init__(settings) install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) @@ -307,7 +307,7 @@ class CrawlerProcess(CrawlerRunner): If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. - :param boolean stop_after_crawl: stop or not the reactor when all + :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished """ from twisted.internet import reactor @@ -340,5 +340,5 @@ class CrawlerProcess(CrawlerRunner): def _handle_twisted_reactor(self): if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"]) + install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) super()._handle_twisted_reactor() diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 77048f389..87f8152a4 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -54,8 +54,8 @@ class CookiesMiddleware: cl = [to_unicode(c, errors='replace') for c in request.headers.getlist('Cookie')] if cl: - cookies = "\n".join("Cookie: {}\n".format(c) for c in cl) - msg = "Sending cookies to: {}\n{}".format(request, cookies) + cookies = "\n".join(f"Cookie: {c}\n" for c in cl) + msg = f"Sending cookies to: {request}\n{cookies}" logger.debug(msg, extra={'spider': spider}) def _debug_set_cookie(self, response, spider): @@ -63,8 +63,8 @@ class CookiesMiddleware: cl = [to_unicode(c, errors='replace') for c in response.headers.getlist('Set-Cookie')] if cl: - cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl) - msg = "Received cookies from: {}\n{}".format(response, cookies) + cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl) + msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={'spider': spider}) def _format_cookie(self, cookie, request): @@ -74,7 +74,7 @@ class CookiesMiddleware: """ decoded = {} for key in ("name", "value", "path", "domain"): - if not cookie.get(key): + if cookie.get(key) is None: if key in ("name", "value"): msg = "Invalid cookie found in request {}: {} ('{}' is missing)" logger.warning(msg.format(request, cookie, key)) @@ -90,9 +90,9 @@ class CookiesMiddleware: request, cookie) decoded[key] = cookie[key].decode("latin1", errors="replace") - cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value")) + cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" for key, value in decoded.items(): # path, domain - cookie_str += "; {}={}".format(key.capitalize(), value) + cookie_str += f"; {key.capitalize()}={value}" return cookie_str def _get_request_cookies(self, jar, request): diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 6db57bd8b..62f1c3a29 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,4 +1,5 @@ from email.utils import formatdate +from typing import Optional, Type, TypeVar from twisted.internet import defer from twisted.internet.error import ( @@ -13,10 +14,19 @@ from twisted.internet.error import ( from twisted.web.client import ResponseFailed from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.http.request import Request +from scrapy.http.response import Response +from scrapy.settings import Settings +from scrapy.spiders import Spider +from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object +HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware") + + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -24,7 +34,7 @@ class HttpCacheMiddleware: ConnectionLost, TCPTimedOutError, ResponseFailed, IOError) - def __init__(self, settings, stats): + def __init__(self, settings: Settings, stats: StatsCollector) -> None: if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings) @@ -33,26 +43,26 @@ class HttpCacheMiddleware: self.stats = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV: o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.storage.open_spider(spider) - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> None: self.storage.close_spider(spider) - def process_request(self, request, spider): + def process_request(self, request: Request, spider: Spider) -> Optional[Response]: if request.meta.get('dont_cache', False): - return + return None # Skip uncacheable requests if not self.policy.should_cache_request(request): request.meta['_dont_cache'] = True # flag as uncacheable - return + return None # Look for cached response and check if expired cachedresponse = self.storage.retrieve_response(spider, request) @@ -61,7 +71,7 @@ class HttpCacheMiddleware: if self.ignore_missing: self.stats.inc_value('httpcache/ignore', spider=spider) raise IgnoreRequest("Ignored request not in cache: %s" % request) - return # first time request + return None # first time request # Return cached response only if not expired cachedresponse.flags.append('cached') @@ -73,7 +83,9 @@ class HttpCacheMiddleware: # process_response hook request.meta['cached_response'] = cachedresponse - def process_response(self, request, response, spider): + return None + + def process_response(self, request: Request, response: Response, spider: Spider) -> Response: if request.meta.get('dont_cache', False): return response @@ -85,7 +97,7 @@ class HttpCacheMiddleware: # RFC2616 requires origin server to set Date header, # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 if 'Date' not in response.headers: - response.headers['Date'] = formatdate(usegmt=1) + response.headers['Date'] = formatdate(usegmt=True) # Do not validate first-hand responses cachedresponse = request.meta.pop('cached_response', None) @@ -102,13 +114,18 @@ class HttpCacheMiddleware: self._cache_response(spider, response, request, cachedresponse) return response - def process_exception(self, request, exception, spider): + def process_exception( + self, request: Request, exception: Exception, spider: Spider + ) -> Optional[Response]: cachedresponse = request.meta.pop('cached_response', None) if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS): self.stats.inc_value('httpcache/errorrecovery', spider=spider) return cachedresponse + return None - def _cache_response(self, spider, response, request, cachedresponse): + def _cache_response( + self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response] + ) -> None: if self.policy.should_cache_response(response, request): self.stats.inc_value('httpcache/store', spider=spider) self.storage.store_response(spider, request, response) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index da89d3e9b..04da11311 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -24,7 +24,7 @@ class HttpProxyMiddleware: def _basic_auth_header(self, username, password): user_pass = to_bytes( - '%s:%s' % (unquote(username), unquote(password)), + f'{unquote(username)}:{unquote(password)}', encoding=self.auth_encoding) return base64.b64encode(user_pass) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 366d60dcb..4053fecc5 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): enabled_setting = 'METAREFRESH_ENABLED' def __init__(self, settings): - super(MetaRefreshMiddleware, self).__init__(settings) + super().__init__(settings) self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS') self._maxdelay = settings.getint('METAREFRESH_MAXDELAY') diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 67be8c282..51fe59254 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -88,7 +88,7 @@ class RetryMiddleware: reason = global_object_name(reason.__class__) stats.inc_value('retry/count') - stats.inc_value('retry/reason_count/%s' % reason) + stats.inc_value(f'retry/reason_count/{reason}') return retryreq else: stats.inc_value('retry/max_reached') diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7f18b2bf2..d6da55535 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -61,7 +61,7 @@ class RobotsTxtMiddleware: if netloc not in self._parsers: self._parsers[netloc] = Deferred() - robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc) + robotsurl = f"{url.scheme}://{url.netloc}/robots.txt" robotsreq = Request( robotsurl, priority=self.DOWNLOAD_PRIORITY, @@ -94,7 +94,7 @@ class RobotsTxtMiddleware: def _parse_robots(self, response, netloc, spider): self.crawler.stats.inc_value('robotstxt/response_count') - self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status)) + self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}') rp = self._parserimpl.from_crawler(self.crawler, response.body) rp_dfd = self._parsers[netloc] self._parsers[netloc] = rp @@ -102,7 +102,7 @@ class RobotsTxtMiddleware: def _robots_error(self, failure, netloc): if failure.type is not IgnoreRequest: - key = 'robotstxt/exception_count/{}'.format(failure.type) + key = f'robotstxt/exception_count/{failure.type}' self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] self._parsers[netloc] = None diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 46a2ad397..5479cd0e2 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -17,13 +17,13 @@ class DownloaderStats: def process_request(self, request, spider): self.stats.inc_value('downloader/request_count', spider=spider) - self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider) + self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider) reqlen = len(request_httprepr(request)) self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider) def process_response(self, request, response, spider): self.stats.inc_value('downloader/response_count', spider=spider) - self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider) + self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider) reslen = len(response_httprepr(response)) self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response @@ -31,4 +31,4 @@ class DownloaderStats: def process_exception(self, request, exception, spider): ex_class = global_object_name(exception.__class__) self.stats.inc_value('downloader/exception_count', spider=spider) - self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) + self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 45f152321..0c410f035 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -37,7 +37,7 @@ class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" def __init__(self, reason='cancelled'): - super(CloseSpider, self).__init__() + super().__init__() self.reason = reason @@ -74,7 +74,7 @@ class UsageError(Exception): def __init__(self, *a, **kw): self.print_help = kw.pop('print_help', True) - super(UsageError, self).__init__(*a, **kw) + super().__init__(*a, **kw) class ScrapyDeprecationWarning(Warning): diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 712572673..fb4b565cf 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -39,7 +39,7 @@ class BaseItemExporter: self.export_empty_fields = options.pop('export_empty_fields', False) self.indent = options.pop('indent', None) if not dont_fail and options: - raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) + raise TypeError(f"Unexpected options: {', '.join(options.keys())}") def export_item(self, item): raise NotImplementedError @@ -195,7 +195,7 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): - def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs): + def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs): super().__init__(dont_fail=True, **kwargs) if not self.encoding: self.encoding = 'utf-8' @@ -205,7 +205,8 @@ class CsvItemExporter(BaseItemExporter): line_buffering=False, write_through=True, encoding=self.encoding, - newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + errors=errors, ) self.csv_writer = csv.writer(self.stream, **self._kwargs) self._headers_not_written = True @@ -243,12 +244,8 @@ class CsvItemExporter(BaseItemExporter): def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: if not self.fields_to_export: - if isinstance(item, dict): - # for dicts try using fields of the first item - self.fields_to_export = list(item.keys()) - else: - # use fields declared in Item - self.fields_to_export = list(item.fields.keys()) + # use declared field names, or keys if the item is a dict + self.fields_to_export = ItemAdapter(item).field_names() row = list(self._build_row(self.fields_to_export)) self.csv_writer.writerow(row) @@ -305,7 +302,7 @@ class PythonItemExporter(BaseItemExporter): def _configure(self, options, dont_fail=False): self.binary = options.pop('binary', True) - super(PythonItemExporter, self)._configure(options, dont_fail) + super()._configure(options, dont_fail) if self.binary: warnings.warn( "PythonItemExporter will drop support for binary export in the future", diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 389cb65bc..675f8276f 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -43,4 +43,4 @@ class CoreStats: def item_dropped(self, item, spider, exception): reason = exception.__class__.__name__ self.stats.inc_value('item_dropped_count', spider=spider) - self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider) + self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 586399784..fd2a02d8d 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -48,7 +48,7 @@ class StackTraceDump: for id_, frame in sys._current_frames().items(): name = id2name.get(id_, '') dump = ''.join(traceback.format_stack(frame)) - dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump) + dumps += f"# Thread: {name}({id_})\n{dump}\n" return dumps diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 72a34ae0d..980825499 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst import logging import os +import re import sys import warnings from datetime import datetime @@ -23,17 +24,34 @@ from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.python import without_none_values +from scrapy.utils.python import get_func_args, without_none_values logger = logging.getLogger(__name__) +def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): + argument_names = get_func_args(builder) + if 'feed_options' in argument_names: + kwargs['feed_options'] = feed_options + else: + warnings.warn( + "{} does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove this " + "warning. This parameter will become mandatory in a future " + "version of Scrapy." + .format(builder.__qualname__), + category=ScrapyDeprecationWarning + ) + return builder(*preargs, uri, *args, **kwargs) + + class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" - def __init__(uri): - """Initialize the storage with the parameters given in the URI""" + def __init__(uri, *, feed_options=None): + """Initialize the storage with the parameters given in the URI and the + feed-specific options (see :setting:`FEEDS`)""" def open(spider): """Open the storage for the given spider. It must return a file-like @@ -63,10 +81,15 @@ class BlockingFeedStorage: @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None): + def __init__(self, uri, _stdout=None, *, feed_options=None): if not _stdout: _stdout = sys.stdout.buffer self._stdout = _stdout + if feed_options and feed_options.get('overwrite', False) is True: + logger.warning('Standard output (stdout) storage does not support ' + 'overwriting. To suppress this warning, remove the ' + 'overwrite option from your FEEDS setting, or set ' + 'it to False.') def open(self, spider): return self._stdout @@ -78,14 +101,16 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri): + def __init__(self, uri, *, feed_options=None): self.path = file_uri_to_path(uri) + feed_options = feed_options or {} + self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab' def open(self, spider): dirname = os.path.dirname(self.path) if dirname and not os.path.exists(dirname): os.makedirs(dirname) - return open(self.path, 'ab') + return open(self.path, self.write_mode) def store(self, file): file.close() @@ -93,24 +118,8 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None, acl=None): - # BEGIN Backward compatibility for initialising without keys (and - # without using from_crawler) - no_defaults = access_key is None and secret_key is None - if no_defaults: - from scrapy.utils.project import get_project_settings - settings = get_project_settings() - if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings: - warnings.warn( - "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " - "without AWS keys is deprecated. Please supply credentials or " - "use the `from_crawler()` constructor.", - category=ScrapyDeprecationWarning, - stacklevel=2 - ) - access_key = settings['AWS_ACCESS_KEY_ID'] - secret_key = settings['AWS_SECRET_ACCESS_KEY'] - # END Backward compatibility + def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, + feed_options=None): u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key @@ -127,14 +136,20 @@ class S3FeedStorage(BlockingFeedStorage): else: import boto self.connect_s3 = boto.connect_s3 + if feed_options and feed_options.get('overwrite', True) is False: + logger.warning('S3 does not support appending to files. To ' + 'suppress this warning, remove the overwrite ' + 'option from your FEEDS setting or set it to True.') @classmethod - def from_crawler(cls, crawler, uri): - return cls( - uri=uri, + def from_crawler(cls, crawler, uri, *, feed_options=None): + return build_storage( + cls, + uri, access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], - acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None + acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, + feed_options=feed_options, ) def _store_in_thread(self, file): @@ -151,6 +166,7 @@ class S3FeedStorage(BlockingFeedStorage): kwargs = {'policy': self.acl} if self.acl else {} key.set_contents_from_file(file, **kwargs) key.close() + file.close() class GCSFeedStorage(BlockingFeedStorage): @@ -181,39 +197,45 @@ class GCSFeedStorage(BlockingFeedStorage): class FTPFeedStorage(BlockingFeedStorage): - def __init__(self, uri, use_active_mode=False): + def __init__(self, uri, use_active_mode=False, *, feed_options=None): u = urlparse(uri) self.host = u.hostname self.port = int(u.port or '21') self.username = u.username - self.password = unquote(u.password) + self.password = unquote(u.password or '') self.path = u.path self.use_active_mode = use_active_mode + self.overwrite = not feed_options or feed_options.get('overwrite', True) @classmethod - def from_crawler(cls, crawler, uri): - return cls( - uri=uri, - use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE') + def from_crawler(cls, crawler, uri, *, feed_options=None): + return build_storage( + cls, + uri, + crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'), + feed_options=feed_options, ) def _store_in_thread(self, file): ftp_store_file( path=self.path, file=file, host=self.host, port=self.port, username=self.username, - password=self.password, use_active_mode=self.use_active_mode + password=self.password, use_active_mode=self.use_active_mode, + overwrite=self.overwrite, ) class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template): self.file = file self.exporter = exporter self.storage = storage # feed params - self.uri = uri + self.batch_id = batch_id self.format = format self.store_empty = store_empty + self.uri_template = uri_template + self.uri = uri # flags self.itemcount = 0 self._exporting = False @@ -256,77 +278,126 @@ class FeedExporter: category=ScrapyDeprecationWarning, stacklevel=2, ) uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects - feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} - self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings) + feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} + self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' - for uri, feed in self.settings.getdict('FEEDS').items(): + for uri, feed_options in self.settings.getdict('FEEDS').items(): uri = str(uri) # handle pathlib.Path objects - self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings) + self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - for uri, feed in self.feeds.items(): - if not self._storage_supported(uri): + for uri, feed_options in self.feeds.items(): + if not self._storage_supported(uri, feed_options): raise NotConfigured - if not self._exporter_supported(feed['format']): + if not self._settings_are_valid(): + raise NotConfigured + if not self._exporter_supported(feed_options['format']): raise NotConfigured def open_spider(self, spider): - for uri, feed in self.feeds.items(): - uri = uri % self._get_uri_params(spider, feed['uri_params']) - storage = self._get_storage(uri) - file = storage.open(spider) - exporter = self._get_exporter( - file=file, - format=feed['format'], - fields_to_export=feed['fields'], - encoding=feed['encoding'], - indent=feed['indent'], - ) - slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) - self.slots.append(slot) - if slot.store_empty: - slot.start_exporting() + for uri, feed_options in self.feeds.items(): + uri_params = self._get_uri_params(spider, feed_options['uri_params']) + self.slots.append(self._start_new_batch( + batch_id=1, + uri=uri % uri_params, + feed_options=feed_options, + spider=spider, + uri_template=uri, + )) def close_spider(self, spider): deferred_list = [] for slot in self.slots: - if not slot.itemcount and not slot.store_empty: - # We need to call slot.storage.store nonetheless to get the file - # properly closed. - d = defer.maybeDeferred(slot.storage.store, slot.file) - deferred_list.append(d) - continue - slot.finish_exporting() - logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" - log_args = {'format': slot.format, - 'itemcount': slot.itemcount, - 'uri': slot.uri} - d = defer.maybeDeferred(slot.storage.store, slot.file) - - # Use `largs=log_args` to copy log_args into function's scope - # instead of using `log_args` from the outer scope - d.addCallback( - lambda _, largs=log_args: logger.info( - logfmt % "Stored", largs, extra={'spider': spider} - ) - ) - d.addErrback( - lambda f, largs=log_args: logger.error( - logfmt % "Error storing", largs, - exc_info=failure_to_exc_info(f), extra={'spider': spider} - ) - ) + d = self._close_slot(slot, spider) deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None + def _close_slot(self, slot, spider): + if not slot.itemcount and not slot.store_empty: + # We need to call slot.storage.store nonetheless to get the file + # properly closed. + return defer.maybeDeferred(slot.storage.store, slot.file) + slot.finish_exporting() + logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" + log_args = {'format': slot.format, + 'itemcount': slot.itemcount, + 'uri': slot.uri} + d = defer.maybeDeferred(slot.storage.store, slot.file) + + # Use `largs=log_args` to copy log_args into function's scope + # instead of using `log_args` from the outer scope + d.addCallback( + lambda _, largs=log_args: logger.info( + logfmt % "Stored", largs, extra={'spider': spider} + ) + ) + d.addErrback( + lambda f, largs=log_args: logger.error( + logfmt % "Error storing", largs, + exc_info=failure_to_exc_info(f), extra={'spider': spider} + ) + ) + return d + + def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): + """ + Redirect the output data stream to a new file. + Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified + :param batch_id: sequence number of current batch + :param uri: uri of the new batch to start + :param feed_options: dict with parameters of feed + :param spider: user spider + :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri + """ + storage = self._get_storage(uri, feed_options) + file = storage.open(spider) + exporter = self._get_exporter( + file=file, + format=feed_options['format'], + fields_to_export=feed_options['fields'], + encoding=feed_options['encoding'], + indent=feed_options['indent'], + ) + slot = _FeedSlot( + file=file, + exporter=exporter, + storage=storage, + uri=uri, + format=feed_options['format'], + store_empty=feed_options['store_empty'], + batch_id=batch_id, + uri_template=uri_template, + ) + if slot.store_empty: + slot.start_exporting() + return slot + def item_scraped(self, item, spider): + slots = [] for slot in self.slots: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 + # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one + if ( + self.feeds[slot.uri_template]['batch_item_count'] + and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] + ): + uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) + self._close_slot(slot, spider) + slots.append(self._start_new_batch( + batch_id=slot.batch_id + 1, + uri=slot.uri_template % uri_params, + feed_options=self.feeds[slot.uri_template], + spider=spider, + uri_template=slot.uri_template, + )) + else: + slots.append(slot) + self.slots = slots def _load_components(self, setting_prefix): conf = without_none_values(self.settings.getwithbase(setting_prefix)) @@ -343,11 +414,27 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) - def _storage_supported(self, uri): + def _settings_are_valid(self): + """ + If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain + %(batch_time)s or %(batch_id)d to distinguish different files of partial output + """ + for uri_template, values in self.feeds.items(): + if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): + logger.error( + '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT ' + 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: ' + 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' + ''.format(uri_template) + ) + return False + return True + + def _storage_supported(self, uri, feed_options): scheme = urlparse(uri).scheme if scheme in self.storages: try: - self._get_storage(uri) + self._get_storage(uri, feed_options) return True except NotConfigured as e: logger.error("Disabled feed storage scheme: %(scheme)s. " @@ -365,15 +452,39 @@ class FeedExporter: def _get_exporter(self, file, format, *args, **kwargs): return self._get_instance(self.exporters[format], file, *args, **kwargs) - def _get_storage(self, uri): - return self._get_instance(self.storages[urlparse(uri).scheme], uri) + def _get_storage(self, uri, feed_options): + """Fork of create_instance specific to feed storage classes - def _get_uri_params(self, spider, uri_params): + It supports not passing the *feed_options* parameters to classes that + do not support it, and issuing a deprecation warning instead. + """ + feedcls = self.storages[urlparse(uri).scheme] + crawler = getattr(self, 'crawler', None) + + def build_instance(builder, *preargs): + return build_storage(builder, uri, preargs=preargs) + + if crawler and hasattr(feedcls, 'from_crawler'): + instance = build_instance(feedcls.from_crawler, crawler) + method_name = 'from_crawler' + elif hasattr(feedcls, 'from_settings'): + instance = build_instance(feedcls.from_settings, self.settings) + method_name = 'from_settings' + else: + instance = build_instance(feedcls) + method_name = '__new__' + if instance is None: + raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name)) + return instance + + def _get_uri_params(self, spider, uri_params, slot=None): params = {} for k in dir(spider): params[k] = getattr(spider, k) - ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') - params['time'] = ts + utc_now = datetime.utcnow() + params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-') + params['batch_time'] = utc_now.isoformat().replace(':', '-') + params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 6294a9b52..e0c04b2de 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -223,7 +223,7 @@ class DbmCacheStorage: self.db = None def open_spider(self, spider): - dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) + dbpath = os.path.join(self.cachedir, f'{spider.name}.db') self.db = self.dbmodule.open(dbpath, 'c') logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) @@ -251,13 +251,13 @@ class DbmCacheStorage: 'headers': dict(response.headers), 'body': response.body, } - self.db['%s_data' % key] = pickle.dumps(data, protocol=4) - self.db['%s_time' % key] = str(time()) + self.db[f'{key}_data'] = pickle.dumps(data, protocol=4) + self.db[f'{key}_time'] = str(time()) def _read_data(self, spider, request): key = self._request_key(request) db = self.db - tkey = '%s_time' % key + tkey = f'{key}_time' if tkey not in db: return # not found @@ -265,7 +265,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return # expired - return pickle.loads(db['%s_data' % key]) + return pickle.loads(db[f'{key}_data']) def _request_key(self, request): return request_fingerprint(request) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index dc8cdbb1d..cee44ea62 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -30,4 +30,4 @@ class MemoryDebugger: for cls, wdict in live_refs.items(): if not wdict: continue - self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider) + self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ab2e43e8c..274cbdbfe 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -82,8 +82,8 @@ class MemoryUsage: {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( - "%s terminated: memory usage exceeded %dM at %s" - % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + f"{self.crawler.settings['BOT_NAME']} terminated: " + f"memory usage exceeded {mem}M at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) @@ -105,8 +105,8 @@ class MemoryUsage: {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( - "%s warning: memory usage reached %dM at %s" - % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + f"{self.crawler.settings['BOT_NAME']} warning: " + f"memory usage reached {mem}M at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/warning_notified', 1) @@ -115,9 +115,9 @@ class MemoryUsage: def _send_report(self, rcpts, subject): """send notification mail with some additional useful info""" stats = self.crawler.stats - s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024) - s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024) - s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024) + s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n" + s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n" + s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n" s += "ENGINE STATUS ------------------------------------------------------- \r\n" s += "\r\n" diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 320f13b29..bcdbaff24 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -24,11 +24,11 @@ class StatsMailer: o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - + def spider_closed(self, spider): spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" - body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items()) - body += "\n\n%s stats\n\n" % spider.name - body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) - return self.mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) + body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) + body += f"\n\n{spider.name} stats\n\n" + body += "\n".join(f"{k:<50} : {v}" for k, v in spider_stats.items()) + return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body) diff --git a/scrapy/http/common.py b/scrapy/http/common.py index ba6ab277c..98699d7fd 100644 --- a/scrapy/http/common.py +++ b/scrapy/http/common.py @@ -1,6 +1,6 @@ def obsolete_setter(setter, attrname): def newsetter(self, value): c = self.__class__.__name__ - msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c) + msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead" raise AttributeError(msg) return newsetter diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index dcaaeddfa..1a2b99b0a 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -8,7 +8,7 @@ class Headers(CaselessDict): def __init__(self, seq=None, encoding='utf-8'): self.encoding = encoding - super(Headers, self).__init__(seq) + super().__init__(seq) def normkey(self, key): """Normalize key to bytes""" @@ -33,23 +33,23 @@ class Headers(CaselessDict): elif isinstance(x, int): return str(x).encode(self.encoding) else: - raise TypeError('Unsupported value type: {}'.format(type(x))) + raise TypeError(f'Unsupported value type: {type(x)}') def __getitem__(self, key): try: - return super(Headers, self).__getitem__(key)[-1] + return super().__getitem__(key)[-1] except IndexError: return None def get(self, key, def_val=None): try: - return super(Headers, self).get(key, def_val)[-1] + return super().get(key, def_val)[-1] except IndexError: return None def getlist(self, key, def_val=None): try: - return super(Headers, self).__getitem__(key) + return super().__getitem__(key) except KeyError: if def_val is not None: return self.normvalue(def_val) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a98ba9960..ef58deacc 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -25,13 +25,13 @@ class Request(object_ref): self._set_url(url) self._set_body(body) if not isinstance(priority, int): - raise TypeError("Request priority not an integer: %r" % priority) + raise TypeError(f"Request priority not an integer: {priority!r}") self.priority = priority if callback is not None and not callable(callback): - raise TypeError('callback must be a callable, got %s' % type(callback).__name__) + raise TypeError(f'callback must be a callable, got {type(callback).__name__}') if errback is not None and not callable(errback): - raise TypeError('errback must be a callable, got %s' % type(errback).__name__) + raise TypeError(f'errback must be a callable, got {type(errback).__name__}') self.callback = callback self.errback = errback @@ -60,13 +60,13 @@ class Request(object_ref): def _set_url(self, url): if not isinstance(url, str): - raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) + raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}') s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) if ('://' not in self._url) and (not self._url.startswith('data:')): - raise ValueError('Missing scheme in request url: %s' % self._url) + raise ValueError(f'Missing scheme in request url: {self._url}') url = property(_get_url, obsolete_setter(_set_url, 'url')) @@ -86,7 +86,7 @@ class Request(object_ref): return self._encoding def __str__(self): - return "<%s %s>" % (self.method, self.url) + return f"<{self.method} {self.url}>" __repr__ = __str__ diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0e6ceef0b..2815303a2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -24,7 +24,7 @@ class FormRequest(Request): if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' - super(FormRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata @@ -80,15 +80,15 @@ def _get_form(response, formname, formid, formnumber, formxpath): base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: - raise ValueError("No element found in %s" % response) + raise ValueError(f"No element found in {response}") if formname is not None: - f = root.xpath('//form[@name="%s"]' % formname) + f = root.xpath(f'//form[@name="{formname}"]') if f: return f[0] if formid is not None: - f = root.xpath('//form[@id="%s"]' % formid) + f = root.xpath(f'//form[@id="{formid}"]') if f: return f[0] @@ -103,7 +103,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): el = el.getparent() if el is None: break - raise ValueError('No element found with %s' % formxpath) + raise ValueError(f'No element found with {formxpath}') # If we get here, it means that either formname was None # or invalid @@ -111,8 +111,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): try: form = forms[formnumber] except IndexError: - raise IndexError("Form number %d not found in %s" % - (formnumber, response)) + raise IndexError(f"Form number {formnumber} not found in {response}") else: return form @@ -133,7 +132,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={ "re": "http://exslt.org/regular-expressions"}) - values = [(k, u'' if v is None else v) + values = [(k, '' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys] @@ -168,7 +167,7 @@ def _select_value(ele, n, v): # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') - v = [(o.get('value') or o.text or u'').strip() for o in selected_options] + v = [(o.get('value') or o.text or '').strip() for o in selected_options] return n, v @@ -205,12 +204,12 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) + xpath = './/*' + ''.join(f'[@{k}="{v}"]' for k, v in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') elif len(el) > 1: - raise ValueError("Multiple elements found (%r) matching the criteria " - "in clickdata: %r" % (el, clickdata)) + raise ValueError(f"Multiple elements found ({el!r}) matching the " + f"criteria in clickdata: {clickdata!r}") else: - raise ValueError('No clickable element matching clickdata: %r' % (clickdata,)) + raise ValueError(f'No clickable element matching clickdata: {clickdata!r}') diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index f08b25280..eae3f9f6b 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -32,7 +32,7 @@ class JsonRequest(Request): if 'method' not in kwargs: kwargs['method'] = 'POST' - super(JsonRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') @@ -47,7 +47,7 @@ class JsonRequest(Request): elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) - return super(JsonRequest, self).replace(*args, **kwargs) + return super().replace(*args, **kwargs) def _dumps(self, data): """Convert to JSON """ diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 811d3ad6b..c70912e49 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -31,5 +31,5 @@ class XmlRpcRequest(Request): if encoding is not None: kwargs['encoding'] = encoding - super(XmlRpcRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'text/xml') diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c2c37dd1d..c635fde69 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -55,8 +55,8 @@ class Response(object_ref): if isinstance(url, str): self._url = url else: - raise TypeError('%s url must be str, got %s:' % - (type(self).__name__, type(url).__name__)) + raise TypeError(f'{type(self).__name__} url must be str, ' + f'got {type(url).__name__}') url = property(_get_url, obsolete_setter(_set_url, 'url')) @@ -77,7 +77,7 @@ class Response(object_ref): body = property(_get_body, obsolete_setter(_set_body, 'body')) def __str__(self): - return "<%d %s>" % (self.status, self.url) + return f"<{self.status} {self.url}>" __repr__ = __str__ diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0f300c8da..e36e14880 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -35,23 +35,23 @@ class TextResponse(Response): self._cached_benc = None self._cached_ubody = None self._cached_selector = None - super(TextResponse, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def _set_url(self, url): if isinstance(url, str): self._url = to_unicode(url, self.encoding) else: - super(TextResponse, self)._set_url(url) + super()._set_url(url) def _set_body(self, body): self._body = b'' # used by encoding detection if isinstance(body, str): if self._encoding is None: - raise TypeError('Cannot convert unicode body - %s has no encoding' % - type(self).__name__) + raise TypeError('Cannot convert unicode body - ' + f'{type(self).__name__} has no encoding') self._body = body.encode(self._encoding) else: - super(TextResponse, self)._set_body(body) + super()._set_body(body) def replace(self, *args, **kwargs): kwargs.setdefault('encoding', self.encoding) @@ -92,7 +92,7 @@ class TextResponse(Response): # _body_inferred_encoding is called benc = self.encoding if self._cached_ubody is None: - charset = 'charset=%s' % benc + charset = f'charset={benc}' self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody @@ -166,7 +166,7 @@ class TextResponse(Response): elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") encoding = self.encoding if encoding is None else encoding - return super(TextResponse, self).follow( + return super().follow( url=url, callback=callback, method=method, @@ -226,7 +226,7 @@ class TextResponse(Response): for sel in selectors: with suppress(_InvalidSelector): urls.append(_url_from_selector(sel)) - return super(TextResponse, self).follow_all( + return super().follow_all( urls=urls, callback=callback, method=method, @@ -255,12 +255,11 @@ def _url_from_selector(sel): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): - raise _InvalidSelector("Unsupported selector: %s" % sel) + raise _InvalidSelector(f"Unsupported selector: {sel}") if sel.root.tag not in ('a', 'link'): - raise _InvalidSelector("Only and elements are supported; got <%s>" % - sel.root.tag) + raise _InvalidSelector("Only and elements are supported; " + f"got <{sel.root.tag}>") href = sel.root.get('href') if href is None: - raise _InvalidSelector("<%s> element has no href attribute: %s" % - (sel.root.tag, sel)) + raise _InvalidSelector(f"<{sel.root.tag}> element has no href attribute: {sel}") return strip_html5_whitespace(href) diff --git a/scrapy/item.py b/scrapy/item.py index 4ab83d1a0..af3849302 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta): if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) - return super(BaseItem, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) class Field(dict): @@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta): def __new__(mcs, class_name, bases, attrs): classcell = attrs.pop('__classcell__', None) new_bases = tuple(base._class for base in bases if hasattr(base, '_class')) - _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs) + _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs) fields = getattr(_class, 'fields', {}) new_attrs = {} @@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta): new_attrs['_class'] = _class if classcell is not None: new_attrs['__classcell__'] = classcell - return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs) + return super().__new__(mcs, class_name, bases, new_attrs) class DictItem(MutableMapping, BaseItem): @@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem): if issubclass(cls, DictItem) and not issubclass(cls, Item): warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) - return super(DictItem, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) def __init__(self, *args, **kwargs): self._values = {} @@ -96,20 +96,20 @@ class DictItem(MutableMapping, BaseItem): if key in self.fields: self._values[key] = value else: - raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key)) + raise KeyError(f"{self.__class__.__name__} does not support field: {key}") def __delitem__(self, key): del self._values[key] def __getattr__(self, name): if name in self.fields: - raise AttributeError("Use item[%r] to get field value" % name) + raise AttributeError(f"Use item[{name!r}] to get field value") raise AttributeError(name) def __setattr__(self, name, value): if not name.startswith('_'): - raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) - super(DictItem, self).__setattr__(name, value) + raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value") + super().__setattr__(name, value) def __len__(self): return len(self._values) diff --git a/scrapy/link.py b/scrapy/link.py index 1ef50b113..684735f6e 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -14,7 +14,7 @@ class Link: def __init__(self, url, text='', fragment='', nofollow=False): if not isinstance(url, str): got = url.__class__.__name__ - raise TypeError("Link urls must be str objects, got %s" % got) + raise TypeError(f"Link urls must be str objects, got {got}") self.url = url self.text = text self.fragment = fragment @@ -33,6 +33,6 @@ class Link: def __repr__(self): return ( - 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' - % (self.url, self.text, self.fragment, self.nofollow) + f'Link(url={self.url!r}, text={self.text!r}, ' + f'fragment={self.fragment!r}, nofollow={self.nofollow!r})' ) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 984a5c4e1..08a6ca1e8 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -65,7 +65,7 @@ class FilteringLinkExtractor: warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) - return super(FilteringLinkExtractor, cls).__new__(cls) + return super().__new__(cls) def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 1615d44d7..e941c4321 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -76,7 +76,7 @@ class LxmlParserLinkExtractor: url = safe_url_string(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) - link = Link(url, _collect_string_content(el) or u'', + link = Link(url, _collect_string_content(el) or '', nofollow=rel_has_nofollow(el.get('rel'))) links.append(link) return self._deduplicate_if_needed(links) @@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): strip=strip, canonicalized=canonicalize ) - super(LxmlLinkExtractor, self).__init__( + super().__init__( link_extractor=lx, allow=allow, deny=deny, diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 219145f13..87568b2d1 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -44,7 +44,7 @@ class LogFormatter: def dropped(self, item, exception, response, spider): return { 'level': logging.INFO, # lowering the level from logging.WARNING - 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s", + 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s", 'args': { 'exception': exception, 'item': item, @@ -54,8 +54,8 @@ class LogFormatter: def crawled(self, request, response, spider): """Logs a message when the crawler finds a webpage.""" - request_flags = ' %s' % str(request.flags) if request.flags else '' - response_flags = ' %s' % str(response.flags) if response.flags else '' + request_flags = f' {str(request.flags)}' if request.flags else '' + response_flags = f' {str(response.flags)}' if response.flags else '' return { 'level': logging.DEBUG, 'msg': CRAWLEDMSG, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 487382a38..99a72aa70 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -108,7 +108,7 @@ class S3FilesStore: from boto.s3.connection import S3Connection self.S3Connection = S3Connection if not uri.startswith("s3://"): - raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri) + raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split('/', 1) def stat_file(self, path, info): @@ -133,7 +133,7 @@ class S3FilesStore: return c.get_bucket(self.bucket, validate=False) def _get_boto_key(self, path): - key_name = '%s%s' % (self.prefix, path) + key_name = f'{self.prefix}{path}' if self.is_botocore: return threads.deferToThread( self.s3_client.head_object, @@ -145,7 +145,7 @@ class S3FilesStore: def persist_file(self, path, buf, info, meta=None, headers=None): """Upload file to S3 storage""" - key_name = '%s%s' % (self.prefix, path) + key_name = f'{self.prefix}{path}' buf.seek(0) if self.is_botocore: extra = self._headers_to_botocore_kwargs(self.HEADERS) @@ -208,8 +208,7 @@ class S3FilesStore: try: kwarg = mapping[key] except KeyError: - raise TypeError( - 'Header "%s" is not supported by botocore' % key) + raise TypeError(f'Header "{key}" is not supported by botocore') else: extra[kwarg] = value return extra @@ -283,7 +282,7 @@ class FTPFilesStore: def __init__(self, uri): if not uri.startswith("ftp://"): - raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri) + raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'") u = urlparse(uri) self.port = u.port self.host = u.hostname @@ -293,7 +292,7 @@ class FTPFilesStore: self.basedir = u.path.rstrip('/') def persist_file(self, path, buf, info, meta=None, headers=None): - path = '%s/%s' % (self.basedir, path) + path = f'{self.basedir}/{path}' return threads.deferToThread( ftp_store_file, path=path, file=buf, host=self.host, port=self.port, username=self.username, @@ -308,10 +307,10 @@ class FTPFilesStore: ftp.login(self.username, self.password) if self.USE_ACTIVE_MODE: ftp.set_pasv(False) - file_path = "%s/%s" % (self.basedir, path) - last_modified = float(ftp.voidcmd("MDTM %s" % file_path)[4:].strip()) + file_path = f"{self.basedir}/{path}" + last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) m = hashlib.md5() - ftp.retrbinary('RETR %s' % file_path, m.update) + ftp.retrbinary(f'RETR {file_path}', m.update) return {'last_modified': last_modified, 'checksum': m.hexdigest()} # The file doesn't exist except Exception: @@ -376,7 +375,7 @@ class FilesPipeline(MediaPipeline): resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD ) - super(FilesPipeline, self).__init__(download_func=download_func, settings=settings) + super().__init__(download_func=download_func, settings=settings) @classmethod def from_settings(cls, settings): @@ -409,7 +408,7 @@ class FilesPipeline(MediaPipeline): store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): def _onsuccess(result): if not result: return # returning None force download @@ -436,7 +435,7 @@ class FilesPipeline(MediaPipeline): checksum = result.get('checksum', None) return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'} - path = self.file_path(request, info=info) + path = self.file_path(request, info=info, item=item) dfd = defer.maybeDeferred(self.store.stat_file, path, info) dfd.addCallbacks(_onsuccess, lambda _: None) dfd.addErrback( @@ -460,7 +459,7 @@ class FilesPipeline(MediaPipeline): raise FileException - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): referer = referer_str(request) if response.status != 200: @@ -492,8 +491,8 @@ class FilesPipeline(MediaPipeline): self.inc_stats(info.spider, status) try: - path = self.file_path(request, response=response, info=info) - checksum = self.file_downloaded(response, request, info) + path = self.file_path(request, response=response, info=info, item=item) + checksum = self.file_downloaded(response, request, info, item=item) except FileException as exc: logger.warning( 'File (error): Error processing file from %(request)s ' @@ -515,15 +514,15 @@ class FilesPipeline(MediaPipeline): def inc_stats(self, spider, status): spider.crawler.stats.inc_value('file_count', spider=spider) - spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider) + spider.crawler.stats.inc_value(f'file_status_count/{status}', spider=spider) # Overridable Interface def get_media_requests(self, item, info): urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u) for u in urls] - def file_downloaded(self, response, request, info): - path = self.file_path(request, response=response, info=info) + def file_downloaded(self, response, request, info, *, item=None): + path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) checksum = md5sum(buf) buf.seek(0) @@ -535,7 +534,7 @@ class FilesPipeline(MediaPipeline): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() media_ext = os.path.splitext(request.url)[1] # Handles empty and wild extensions by trying to guess the @@ -545,4 +544,4 @@ class FilesPipeline(MediaPipeline): media_type = mimetypes.guess_type(request.url)[0] if media_type: media_ext = mimetypes.guess_extension(media_type) - return 'full/%s%s' % (media_guid, media_ext) + return f'full/{media_guid}{media_ext}' diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 46f2bfb58..aafd1d8b2 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): - super(ImagesPipeline, self).__init__(store_uri, settings=settings, - download_func=download_func) + super().__init__(store_uri, settings=settings, download_func=download_func) if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -104,12 +103,12 @@ class ImagesPipeline(FilesPipeline): store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) - def file_downloaded(self, response, request, info): - return self.image_downloaded(response, request, info) + def file_downloaded(self, response, request, info, *, item=None): + return self.image_downloaded(response, request, info, item=item) - def image_downloaded(self, response, request, info): + def image_downloaded(self, response, request, info, *, item=None): checksum = None - for path, image, buf in self.get_images(response, request, info): + for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) checksum = md5sum(buf) @@ -120,14 +119,15 @@ class ImagesPipeline(FilesPipeline): headers={'Content-Type': 'image/jpeg'}) return checksum - def get_images(self, response, request, info): - path = self.file_path(request, response=response, info=info) + def get_images(self, response, request, info, *, item=None): + path = self.file_path(request, response=response, info=info, item=item) orig_image = Image.open(BytesIO(response.body)) width, height = orig_image.size if width < self.min_width or height < self.min_height: - raise ImageException("Image too small (%dx%d < %dx%d)" % - (width, height, self.min_width, self.min_height)) + raise ImageException("Image too small " + f"({width}x{height} < " + f"{self.min_width}x{self.min_height})") image, buf = self.convert_image(orig_image) yield path, image, buf @@ -167,10 +167,10 @@ class ImagesPipeline(FilesPipeline): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - return 'full/%s.jpg' % (image_guid) + return f'full/{image_guid}.jpg' def thumb_path(self, request, thumb_id, response=None, info=None): thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid) + return f'thumbs/{thumb_id}/{thumb_guid}.jpg' diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index aa65f4f0e..0a12f3e2c 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,12 +1,16 @@ import functools import logging from collections import defaultdict +from inspect import signature +from warnings import warn + from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter from scrapy.utils.log import failure_to_exc_info @@ -27,6 +31,7 @@ class MediaPipeline: def __init__(self, download_func=None, settings=None): self.download_func = download_func + self._expects_item = {} if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -38,6 +43,9 @@ class MediaPipeline: ) self._handle_statuses(self.allow_redirects) + # Check if deprecated methods are being used and make them compatible + self._make_compatible() + def _handle_statuses(self, allow_redirects): self.handle_httpstatus_list = None if allow_redirects: @@ -53,7 +61,7 @@ class MediaPipeline: 'MYPIPE_IMAGES' """ class_name = self.__class__.__name__ - formatted_key = "{}_{}".format(class_name.upper(), key) + formatted_key = f"{class_name.upper()}_{key}" if ( not base_class_name or class_name == base_class_name @@ -77,11 +85,11 @@ class MediaPipeline: def process_item(self, item, spider): info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) - dlist = [self._process_request(r, info) for r in requests] + dlist = [self._process_request(r, info, item) for r in requests] dfd = DeferredList(dlist, consumeErrors=1) return dfd.addCallback(self.item_completed, item, info) - def _process_request(self, request, info): + def _process_request(self, request, info, item): fp = request_fingerprint(request) cb = request.callback or (lambda _: _) eb = request.errback @@ -102,34 +110,72 @@ class MediaPipeline: # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd = mustbe_deferred(self.media_to_download, request, info) - dfd.addCallback(self._check_media_to_download, request, info) + dfd = mustbe_deferred(self.media_to_download, request, info, item=item) + dfd.addCallback(self._check_media_to_download, request, info, item=item) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) dfd.addErrback(lambda f: logger.error( f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider}) ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _make_compatible(self): + """Make overridable methods of MediaPipeline and subclasses backwards compatible""" + methods = [ + "file_path", "media_to_download", "media_downloaded", + "file_downloaded", "image_downloaded", "get_images" + ] + + for method_name in methods: + method = getattr(self, method_name, None) + if callable(method): + setattr(self, method_name, self._compatible(method)) + + def _compatible(self, func): + """Wrapper for overridable methods to allow backwards compatibility""" + self._check_signature(func) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if self._expects_item[func.__name__]: + return func(*args, **kwargs) + + kwargs.pop('item', None) + return func(*args, **kwargs) + + return wrapper + + def _check_signature(self, func): + sig = signature(func) + self._expects_item[func.__name__] = True + + if 'item' not in sig.parameters: + old_params = str(sig)[1:-1] + new_params = old_params + ", *, item=None" + warn(f'{func.__name__}(self, {old_params}) is deprecated, ' + f'please use {func.__name__}(self, {new_params})', + ScrapyDeprecationWarning, stacklevel=2) + self._expects_item[func.__name__] = False + def _modify_media_request(self, request): if self.handle_httpstatus_list: request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list else: request.meta['handle_httpstatus_all'] = True - def _check_media_to_download(self, result, request, info): + def _check_media_to_download(self, result, request, info, item): if result is not None: return result if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), + callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) else: self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), + callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) return dfd @@ -171,7 +217,7 @@ class MediaPipeline: defer_result(result).chainDeferred(wad) # Overridable Interface - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): """Check request before starting download""" pass @@ -179,7 +225,7 @@ class MediaPipeline: """Returns the media requests to download""" pass - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): """Handler for success downloads""" return response @@ -199,3 +245,7 @@ class MediaPipeline: extra={'spider': info.spider} ) return item + + def file_path(self, request, response=None, info=None, *, item=None): + """Returns the path where downloaded media should be stored""" + pass diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e13d389ee..a9aa6c649 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -141,17 +141,16 @@ class DownloaderAwarePriorityQueue: def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0: - raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP' - % (self.__class__,)) + raise ValueError(f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP') if slot_startprios and not isinstance(slot_startprios, dict): raise ValueError("DownloaderAwarePriorityQueue accepts " - "``slot_startprios`` as a dict; %r instance " + "``slot_startprios`` as a dict; " + f"{slot_startprios.__class__!r} instance " "is passed. Most likely, it means the state is" "created by an incompatible priority queue. " "Only a crawl started with the same priority " - "queue class can be resumed." % - slot_startprios.__class__) + "queue class can be resumed.") self._downloader_interface = DownloaderInterface(crawler) self.downstream_queue_cls = downstream_queue_cls diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f69894b1e..f191deac6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver): """ def __init__(self, reactor, cache_size, timeout): - super(CachingThreadedResolver, self).__init__(reactor) + super().__init__(reactor) dnscache.limit = cache_size self.timeout = timeout @@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver): # so the input argument above is simply overridden # to enforce Scrapy's DNS_TIMEOUT setting's value timeout = (self.timeout,) - d = super(CachingThreadedResolver, self).getHostByName(name, timeout) + d = super().getHostByName(name, timeout) if dnscache.limit: d.addCallback(self._cache_result, name) return d @@ -80,16 +80,16 @@ class CachingHostnameResolver: class CachingResolutionReceiver(resolutionReceiver): def resolutionBegan(self, resolution): - super(CachingResolutionReceiver, self).resolutionBegan(resolution) + super().resolutionBegan(resolution) self.resolution = resolution self.resolved = False def addressResolved(self, address): - super(CachingResolutionReceiver, self).addressResolved(address) + super().addressResolved(address) self.resolved = True def resolutionComplete(self): - super(CachingResolutionReceiver, self).resolutionComplete() + super().resolutionComplete() if self.resolved: dnscache[hostName] = self.resolution diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index d207088e6..6ed9f8b8f 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -45,7 +45,7 @@ class ResponseTypes: elif mimetype in self.classes: return self.classes[mimetype] else: - basetype = "%s/*" % mimetype.split('/')[0] + basetype = f"{mimetype.split('/')[0]}/*" return self.classes.get(basetype, Response) def from_content_type(self, content_type, content_encoding=None): diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 52cf09844..f8649e56b 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -47,10 +47,10 @@ class RobotParser(metaclass=ABCMeta): """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL - :type url: string + :type url: str :param user_agent: User agent - :type user_agent: string + :type user_agent: str """ pass diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 85a9bb526..a25871433 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -66,8 +66,8 @@ class Selector(_ParselSelector, object_ref): def __init__(self, response=None, text=None, type=None, root=None, **kwargs): if response is not None and text is not None: - raise ValueError('%s.__init__() received both response and text' - % self.__class__.__name__) + raise ValueError(f'{self.__class__.__name__}.__init__() received ' + 'both response and text') st = _st(response, type or self._default_type) @@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref): kwargs.setdefault('base_url', response.url) self.response = response - super(Selector, self).__init__(text=text, type=st, root=root, **kwargs) + super().__init__(text=text, type=st, root=root, **kwargs) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index ff8317cd1..1fe1e6fd1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -52,7 +52,7 @@ class SettingsAttribute: self.priority = priority def __str__(self): - return "".format(self=self) + return f"" __repr__ = __str__ @@ -82,7 +82,8 @@ class BaseSettings(MutableMapping): def __init__(self, values=None, priority='project'): self.frozen = False self.attributes = {} - self.update(values, priority) + if values: + self.update(values, priority) def __getitem__(self, opt_name): if opt_name not in self: @@ -97,10 +98,10 @@ class BaseSettings(MutableMapping): Get a setting value without affecting its original type. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any + :type default: object """ return self[name] if self[name] is not None else default @@ -115,10 +116,10 @@ class BaseSettings(MutableMapping): ``'0'`` will return ``False`` when using this method. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any + :type default: object """ got = self.get(name, default) try: @@ -137,10 +138,10 @@ class BaseSettings(MutableMapping): Get a setting value as an int. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any + :type default: object """ return int(self.get(name, default)) @@ -149,10 +150,10 @@ class BaseSettings(MutableMapping): Get a setting value as a float. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any + :type default: object """ return float(self.get(name, default)) @@ -165,10 +166,10 @@ class BaseSettings(MutableMapping): ``'one,two'`` will return a list ['one', 'two'] when using this method. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any + :type default: object """ value = self.get(name, default or []) if isinstance(value, str): @@ -186,10 +187,10 @@ class BaseSettings(MutableMapping): and losing all information about priority and mutability. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any + :type default: object """ value = self.get(name, default or {}) if isinstance(value, str): @@ -201,7 +202,7 @@ class BaseSettings(MutableMapping): counterpart. :param name: name of the dictionary-like setting - :type name: string + :type name: str """ compbs = BaseSettings() compbs.update(self[name + '_BASE']) @@ -214,7 +215,7 @@ class BaseSettings(MutableMapping): the given ``name`` does not exist. :param name: the setting name - :type name: string + :type name: str """ if name not in self: return None @@ -244,14 +245,14 @@ class BaseSettings(MutableMapping): otherwise they won't have any effect. :param name: the setting name - :type name: string + :type name: str :param value: the value to associate with the setting - :type value: any + :type value: object :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() priority = get_settings_priority(priority) @@ -275,11 +276,11 @@ class BaseSettings(MutableMapping): uppercase variable of ``module`` with the provided ``priority``. :param module: the module or the path of the module - :type module: module object or string + :type module: types.ModuleType or str :param priority: the priority of the settings. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() if isinstance(module, str): @@ -308,7 +309,7 @@ class BaseSettings(MutableMapping): :param priority: the priority of the settings. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() if isinstance(values, str): @@ -439,7 +440,7 @@ class Settings(BaseSettings): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user - super(Settings, self).__init__() + super().__init__() self.setmodule(default_settings, 'default') # Promote default dictionaries to BaseSettings instances for per-key # priorities diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9f3d285dc..948b59720 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -19,6 +19,8 @@ from os.path import join, abspath, dirname AJAXCRAWL_ENABLED = False +ASYNCIO_EVENT_LOOP = None + AUTOTHROTTLE_ENABLED = False AUTOTHROTTLE_DEBUG = False AUTOTHROTTLE_MAX_DELAY = 60.0 @@ -147,6 +149,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', } +FEED_EXPORT_BATCH_ITEM_COUNT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', @@ -285,7 +288,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) URLLENGTH_LIMIT = 2083 -USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__version__ +USER_AGENT = f'Scrapy/{import_module("scrapy").__version__} (+https://scrapy.org)' TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/shell.py b/scrapy/shell.py index 10de119ce..c370ccaff 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -140,7 +140,7 @@ class Shell: b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)") for k, v in sorted(self.vars.items()): if self._is_relevant(v): - b.append(" %-10s %s" % (k, v)) + b.append(f" {k:<10} {v}") b.append("Useful shortcuts:") if self.inthread: b.append(" fetch(url[, redirect=True]) " @@ -150,7 +150,7 @@ class Shell: b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") - return "\n".join("[s] %s" % line for line in b) + return "\n".join(f"[s] {line}" for line in b) def _is_relevant(self, value): return isinstance(value, self.relevant_classes) or is_item(value) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 54eb7cfa3..ac4044c64 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -16,7 +16,7 @@ class SignalManager: section. :param receiver: the function to be connected - :type receiver: callable + :type receiver: collections.abc.Callable :param signal: the signal to connect to :type signal: object diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index e11553b39..d7a475b61 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -28,7 +28,7 @@ class SpiderLoader: dupes = [] for name, locations in self._found.items(): dupes.extend([ - " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) + f" {cls} named {name!r} (in {mod})" for mod, cls in locations if len(locations) > 1 ]) @@ -37,7 +37,7 @@ class SpiderLoader: dupes_string = "\n\n".join(dupes) warnings.warn( "There are several spiders with the same name:\n\n" - "{}\n\n This can cause unexpected behavior.".format(dupes_string), + f"{dupes_string}\n\n This can cause unexpected behavior.", category=UserWarning, ) @@ -57,10 +57,9 @@ class SpiderLoader: except ImportError: if self.warn_only: warnings.warn( - "\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format( - modname=name, tb=traceback.format_exc() - ), + f"\n{traceback.format_exc()}Could not load spiders " + f"from module '{name}'. " + "See above traceback for details.", category=RuntimeWarning, ) else: @@ -79,7 +78,7 @@ class SpiderLoader: try: return self._spiders[spider_name] except KeyError: - raise KeyError("Spider not found: {}".format(spider_name)) + raise KeyError(f"Spider not found: {spider_name}") def find_by_request(self, request): """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index fa7f5bef9..776a6879a 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -43,7 +43,7 @@ class DepthMiddleware: return False else: if self.verbose_stats: - self.stats.inc_value('request_depth_count/%s' % depth, + self.stats.inc_value(f'request_depth_count/{depth}', spider=spider) self.stats.max_value('request_depth_max', depth, spider=spider) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 375042340..ae5c258df 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -15,7 +15,7 @@ class HttpError(IgnoreRequest): def __init__(self, response, *args, **kwargs): self.response = response - super(HttpError, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class HttpErrorMiddleware: @@ -48,7 +48,7 @@ class HttpErrorMiddleware: if isinstance(exception, HttpError): spider.crawler.stats.inc_value('httperror/response_ignored_count') spider.crawler.stats.inc_value( - 'httperror/response_ignored_status_count/%s' % response.status + f'httperror/response_ignored_status_count/{response.status}' ) logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index a006f3177..6e4efda97 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -61,15 +61,15 @@ class OffsiteMiddleware: continue elif url_pattern.match(domain): message = ("allowed_domains accepts only domains, not URLs. " - "Ignoring URL entry %s in allowed_domains." % domain) + f"Ignoring URL entry {domain} in allowed_domains.") warnings.warn(message, URLWarning) elif port_pattern.search(domain): message = ("allowed_domains accepts only domains without ports. " - "Ignoring entry %s in allowed_domains." % domain) + f"Ignoring entry {domain} in allowed_domains.") warnings.warn(message, PortWarning) else: domains.append(re.escape(domain)) - regex = r'^(.*\.)?(%s)$' % '|'.join(domains) + regex = fr'^(.*\.)?({"|".join(domains)})$' return re.compile(regex) def spider_opened(self, spider): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 434067b00..f81041376 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -278,7 +278,7 @@ def _load_policy_class(policy, warning_only=False): try: return _policy_classes[policy.lower()] except KeyError: - msg = "Could not load referrer policy %r" % policy + msg = f"Could not load referrer policy {policy!r}" if not warning_only: raise RuntimeError(msg) else: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 856410e6a..30af09124 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -5,6 +5,7 @@ See documentation in docs/topics/spiders.rst """ import logging import warnings +from typing import Optional from scrapy import signals from scrapy.http import Request @@ -32,8 +33,8 @@ class Spider(object_ref): class. """ - name = None - custom_settings = None + name: Optional[str] = None + custom_settings: Optional[dict] = None def __init__(self, name=None, **kwargs): if name is not None: @@ -82,9 +83,8 @@ class Spider(object_ref): warnings.warn( "Spider.make_requests_from_url method is deprecated; it " "won't be called in future Scrapy releases. Please " - "override Spider.start_requests method instead (see %s.%s)." % ( - cls.__module__, cls.__name__ - ), + "override Spider.start_requests method instead " + f"(see {cls.__module__}.{cls.__name__}).", ) for url in self.start_urls: yield self.make_requests_from_url(url) @@ -106,7 +106,7 @@ class Spider(object_ref): return self.parse(response, **kwargs) def parse(self, response, **kwargs): - raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) + raise NotImplementedError(f'{self.__class__.__name__}.parse callback is not defined') @classmethod def update_settings(cls, settings): @@ -123,7 +123,7 @@ class Spider(object_ref): return closed(reason) def __str__(self): - return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self)) + return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" __repr__ = __str__ diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 27c631b8a..02fd3058c 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -7,6 +7,7 @@ See documentation in docs/topics/spiders.rst import copy import warnings +from typing import Sequence from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse @@ -73,10 +74,10 @@ class Rule: @basespider class CrawlSpider(Spider): - rules = () + rules: Sequence[Rule] = () def __init__(self, *a, **kw): - super(CrawlSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self._compile_rules() def _parse(self, response, **kwargs): @@ -146,6 +147,6 @@ class CrawlSpider(Spider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): - spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs) + spider = super().from_crawler(crawler, *args, **kwargs) spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) return spider diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index bf8daa47b..6e0812404 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -72,11 +72,11 @@ class XMLFeedSpider(Spider): elif self.iterator == 'xml': selector = Selector(response, type='xml') self._register_namespaces(selector) - nodes = selector.xpath('//%s' % self.itertag) + nodes = selector.xpath(f'//{self.itertag}') elif self.iterator == 'html': selector = Selector(response, type='html') self._register_namespaces(selector) - nodes = selector.xpath('//%s' % self.itertag) + nodes = selector.xpath(f'//{self.itertag}') else: raise NotSupported('Unsupported node iterator') diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 92dae52b3..396ca5184 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -7,7 +7,7 @@ class InitSpider(Spider): """Base Spider with initialization facilities""" def start_requests(self): - self._postinit_reqs = super(InitSpider, self).start_requests() + self._postinit_reqs = super().start_requests() return iterate_spider_output(self.init_request()) def initialized(self, response=None): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 1d6fbaabe..678651e43 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -19,7 +19,7 @@ class SitemapSpider(Spider): sitemap_alternate_links = False def __init__(self, *a, **kw): - super(SitemapSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: if isinstance(c, str): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index c7ad4d53d..77ffda6f7 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -20,7 +20,7 @@ def _with_mkdir(queue_class): if not os.path.exists(dirname): os.makedirs(dirname, exist_ok=True) - super(DirectoriesCreated, self).__init__(path, *args, **kwargs) + super().__init__(path, *args, **kwargs) return DirectoriesCreated @@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize): def push(self, obj): s = serialize(obj) - super(SerializableQueue, self).push(s) + super().push(s) def pop(self): - s = super(SerializableQueue, self).pop() + s = super().pop() if s: return deserialize(s) @@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class): def __init__(self, crawler, key): self.spider = crawler.spider - super(ScrapyRequestQueue, self).__init__(key) + super().__init__(key) @classmethod def from_crawler(cls, crawler, key, *args, **kwargs): @@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class): def push(self, request): request = request_to_dict(request, self.spider) - return super(ScrapyRequestQueue, self).push(request) + return super().push(request) def pop(self): - request = super(ScrapyRequestQueue, self).pop() + request = super().pop() if not request: return None diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 579c60180..ba7d1a6bf 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -54,7 +54,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): def __init__(self, crawler): - super(MemoryStatsCollector, self).__init__(crawler) + super().__init__(crawler) self.spider_stats = {} def _persist_stats(self, stats, spider): diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index f595a1acb..86238c4cd 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -21,8 +21,8 @@ class Root(Resource): for nl in nlist: args['n'] = nl argstr = urlencode(args, doseq=True) - request.write("follow {1}
    " - .format(argstr, nl).encode('utf8')) + request.write(f"follow {nl}
    " + .encode('utf8')) request.write(b"") return b'' @@ -39,6 +39,6 @@ if __name__ == '__main__': def _print_listening(): httpHost = httpPort.getHost() - print("Bench server at http://{}:{}".format(httpHost.host, httpHost.port)) + print(f"Bench server at http://{httpHost.host}:{httpHost.port}") reactor.callWhenRunning(_print_listening) reactor.run() diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 728bb5f1b..4e7a9967e 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -17,8 +17,8 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _check_components(complist): if len({convert(c) for c in complist}) != len(complist): - raise ValueError('Some paths in {!r} convert to the same object, ' - 'please update your settings'.format(complist)) + raise ValueError(f'Some paths in {complist!r} convert to the same object, ' + 'please update your settings') def _map_keys(compdict): if isinstance(compdict, BaseSettings): @@ -26,9 +26,10 @@ def build_component_list(compdict, custom=None, convert=update_classpath): for k, v in compdict.items(): prio = compdict.getpriority(k) if compbs.getpriority(convert(k)) == prio: - raise ValueError('Some paths in {!r} convert to the same ' + raise ValueError(f'Some paths in {list(compdict.keys())!r} ' + 'convert to the same ' 'object, please update your settings' - ''.format(list(compdict.keys()))) + ) else: compbs.set(convert(k), v, priority=prio) return compbs @@ -40,8 +41,8 @@ def build_component_list(compdict, custom=None, convert=update_classpath): """Fail if a value in the components dict is not a real number or None.""" for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): - raise ValueError('Invalid value {} for component {}, please provide ' - 'a real number or None instead'.format(value, name)) + raise ValueError(f'Invalid value {value} for component {name}, ' + 'please provide a real number or None instead') # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): @@ -115,6 +116,7 @@ def get_sources(use_closest=True): def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() + out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) @@ -126,7 +128,8 @@ def feed_complete_default_values_from_settings(feed, settings): return out -def feed_process_params_from_cli(settings, output, output_format=None): +def feed_process_params_from_cli(settings, output, output_format=None, + overwrite_output=None): """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary @@ -138,22 +141,37 @@ def feed_process_params_from_cli(settings, output, output_format=None): def check_valid_format(output_format): if output_format not in valid_output_formats: - raise UsageError("Unrecognized output format '%s', set one after a" - " colon using the -o option (i.e. -o :)" - " or as a file extension, from the supported list %s" % - (output_format, tuple(valid_output_formats))) + raise UsageError( + f"Unrecognized output format '{output_format}'. " + f"Set a supported one ({tuple(valid_output_formats)}) " + "after a colon at the end of the output URI (i.e. -o/-O " + ":) or as a file extension." + ) + + overwrite = False + if overwrite_output: + if output: + raise UsageError( + "Please use only one of -o/--output and -O/--overwrite-output" + ) + output = overwrite_output + overwrite = True if output_format: if len(output) == 1: check_valid_format(output_format) - warnings.warn('The -t command line option is deprecated in favor' - ' of specifying the output format within the -o' - ' option, please check the -o option docs for more details', - category=ScrapyDeprecationWarning, stacklevel=2) + message = ( + 'The -t command line option is deprecated in favor of ' + 'specifying the output format within the output URI. See the ' + 'documentation of the -o and -O options for more information.', + ) + warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} else: - raise UsageError('The -t command line option cannot be used if multiple' - ' output files are specified with the -o option') + raise UsageError( + 'The -t command-line option cannot be used if multiple output ' + 'URIs are specified' + ) result = {} for element in output: @@ -167,8 +185,10 @@ def feed_process_params_from_cli(settings, output, output_format=None): feed_uri = 'stdout:' check_valid_format(feed_format) result[feed_uri] = {'format': feed_format} + if overwrite: + result[feed_uri]['overwrite'] = True - # FEEDS setting should take precedence over the -o and -t CLI options + # FEEDS setting should take precedence over the matching CLI options result.update(settings.getdict('FEEDS')) return result diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index aa681522f..6660b9dc0 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header class CurlParser(argparse.ArgumentParser): def error(self, message): - error_msg = 'There was an error parsing the curl command: {}'.format(message) + error_msg = f'There was an error parsing the curl command: {message}' raise ValueError(error_msg) @@ -39,7 +39,8 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): :param str curl_command: string containing the curl command :param bool ignore_unknown_options: If true, only a warning is emitted when - cURL options are unknown. Otherwise raises an error. (default: True) + cURL options are unknown. Otherwise + raises an error. (default: True) :return: dictionary of Request kwargs """ @@ -51,7 +52,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) if argv: - msg = 'Unrecognized options: {}'.format(', '.join(argv)) + msg = f'Unrecognized options: {", ".join(argv)}' if ignore_unknown_options: warnings.warn(msg) else: diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 2a92d0588..e31284a7f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -15,7 +15,7 @@ class CaselessDict(dict): __slots__ = () def __init__(self, seq=None): - super(CaselessDict, self).__init__() + super().__init__() if seq: self.update(seq) @@ -53,7 +53,7 @@ class CaselessDict(dict): def update(self, seq): seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) - super(CaselessDict, self).update(iseq) + super().update(iseq) @classmethod def fromkeys(cls, keys, value=None): @@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict): """ def __init__(self, limit=None): - super(LocalCache, self).__init__() + super().__init__() self.limit = limit def __setitem__(self, key, value): if self.limit: while len(self) >= self.limit: self.popitem(last=False) - super(LocalCache, self).__setitem__(key, value) + super().__setitem__(key, value) class LocalWeakReferencedCache(weakref.WeakKeyDictionary): @@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): """ def __init__(self, limit=None): - super(LocalWeakReferencedCache, self).__init__() + super().__init__() self.data = LocalCache(limit=limit) def __setitem__(self, key, value): try: - super(LocalWeakReferencedCache, self).__setitem__(key, value) + super().__setitem__(key, value) except TypeError: pass # key is not weak-referenceable, skip caching def __getitem__(self, key): try: - return super(LocalWeakReferencedCache, self).__getitem__(key) + return super().__getitem__(key) except (TypeError, KeyError): return None # key is either not weak-referenceable or not cached diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 2e2c7adc1..fef3882cb 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -14,9 +14,9 @@ def deprecated(use_instead=None): def deco(func): @wraps(func) def wrapped(*args, **kwargs): - message = "Call to deprecated function %s." % func.__name__ + message = f"Call to deprecated function {func.__name__}." if use_instead: - message += " Use %s instead." % use_instead + message += f" Use {use_instead} instead." warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) return func(*args, **kwargs) return wrapped diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index a3950db75..21ba02a0b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -124,18 +124,11 @@ def iter_errback(iterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) -def _isfuture(o): - # workaround for Python before 3.5.3 not having asyncio.isfuture - if hasattr(asyncio, 'isfuture'): - return asyncio.isfuture(o) - return isinstance(o, asyncio.Future) - - def deferred_from_coro(o): """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, defer.Deferred): return o - if _isfuture(o) or inspect.isawaitable(o): + if asyncio.isfuture(o) or inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" @@ -167,7 +160,7 @@ def maybeDeferred_coro(f, *args, **kw): if isinstance(result, defer.Deferred): return result - elif _isfuture(result) or inspect.isawaitable(result): + elif asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) elif isinstance(result, failure.Failure): return defer.fail(result) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3dbea5fee..f5b17416f 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -8,9 +8,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning def attribute(obj, oldattr, newattr, version='0.12'): cname = obj.__class__.__name__ warnings.warn( - "%s.%s attribute is deprecated and will be no longer supported " - "in Scrapy %s, use %s.%s attribute instead" - % (cname, oldattr, version, cname, newattr), + f"{cname}.{oldattr} attribute is deprecated and will be no longer supported " + f"in Scrapy {version}, use {cname}.{newattr} attribute instead", ScrapyDeprecationWarning, stacklevel=3) @@ -57,7 +56,7 @@ def create_deprecated_class( warned_on_subclass = False def __new__(metacls, name, bases, clsdict_): - cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_) + cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls @@ -73,7 +72,7 @@ def create_deprecated_class( if warn_once: msg += ' (warning only on first subclass, there may be others)' warnings.warn(msg, warn_category, stacklevel=2) - super(DeprecatedClass, cls).__init__(name, bases, clsdict_) + super().__init__(name, bases, clsdict_) # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks @@ -88,7 +87,7 @@ def create_deprecated_class( # is the deprecated class itself - subclasses of the # deprecated class should not use custom `__subclasscheck__` # method. - return super(DeprecatedClass, cls).__subclasscheck__(sub) + return super().__subclasscheck__(sub) if not inspect.isclass(sub): raise TypeError("issubclass() arg 1 must be a class") @@ -102,7 +101,7 @@ def create_deprecated_class( msg = instance_warn_message.format(cls=_clspath(cls, old_class_path), new=_clspath(new_class, new_class_path)) warnings.warn(msg, warn_category, stacklevel=2) - return super(DeprecatedClass, cls).__call__(*args, **kwargs) + return super().__call__(*args, **kwargs) deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {}) @@ -116,7 +115,7 @@ def create_deprecated_class( # deprecated class is in jinja2 template). __module__ attribute is not # important enough to raise an exception as users may be unable # to fix inspect.stack() errors. - warnings.warn("Error detecting parent module: %r" % e) + warnings.warn(f"Error detecting parent module: {e!r}") return deprecated_cls @@ -124,7 +123,7 @@ def create_deprecated_class( def _clspath(cls, forced=None): if forced is not None: return forced - return '{}.{}'.format(cls.__module__, cls.__name__) + return f'{cls.__module__}.{cls.__name__}' DEPRECATION_RULES = [ @@ -135,9 +134,9 @@ DEPRECATION_RULES = [ def update_classpath(path): """Update a deprecated path from an object with its new location""" for prefix, replacement in DEPRECATION_RULES: - if path.startswith(prefix): + if isinstance(path, str) and path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) - warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path), + warnings.warn(f"`{path}` class is deprecated, use `{new_path}` instead", ScrapyDeprecationWarning) return new_path return path diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 267c7ecd1..0c1cee1a0 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -29,7 +29,7 @@ def get_engine_status(engine): try: checks += [(test, eval(test))] except Exception as e: - checks += [(test, "%s (exception)" % type(e).__name__)] + checks += [(test, f"{type(e).__name__} (exception)")] return checks @@ -38,7 +38,7 @@ def format_engine_status(engine=None): checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: - s += "%-47s : %s\n" % (test, result) + s += f"{test:<47} : {result}\n" s += "\n" return s diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index f07bdd748..6cace4f07 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -20,7 +20,7 @@ def ftp_makedirs_cwd(ftp, path, first_call=True): def ftp_store_file( *, path, file, host, port, - username, password, use_active_mode=False): + username, password, use_active_mode=False, overwrite=True): """Opens a FTP connection with passed credentials,sets current directory to the directory extracted from given path, then uploads the file to server """ @@ -32,4 +32,6 @@ def ftp_store_file( file.seek(0) dirname, filename = posixpath.split(path) ftp_makedirs_cwd(ftp, dirname) - ftp.storbinary('STOR %s' % filename, file) + command = 'STOR' if overwrite else 'APPE' + ftp.storbinary(f'{command} {filename}', file) + file.close() diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index fbd7bd18f..11d433cf5 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -6,11 +6,10 @@ import struct from scrapy.utils.decorators import deprecated -# - Python>=3.5 GzipFile's read() has issues returning leftover -# uncompressed data when input is corrupted -# (regression or bug-fix compared to Python 3.4) +# - GzipFile's read() has issues returning leftover uncompressed data when +# input is corrupted # - read1(), which fetches data before raising EOFError on next call -# works here but is only available from Python>=3.3 +# works here @deprecated('GzipFile.read1') def read1(gzf, size=-1): return gzf.read1(size) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 5e15bf0c8..789da1392 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -22,8 +22,8 @@ def xmliter(obj, nodename): """ nodename_patt = re.escape(nodename) - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S) - HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S) + HEADER_START_RE = re.compile(fr'^(.*?)<\s*{nodename_patt}(?:\s|>)', re.S) + HEADER_END_RE = re.compile(fr'<\s*/{nodename_patt}\s*>', re.S) text = _body_or_str(obj) header_start = re.search(HEADER_START_RE, text) @@ -31,7 +31,7 @@ def xmliter(obj, nodename): header_end = re_rsearch(HEADER_END_RE, text) header_end = text[header_end[1]:].strip() if header_end else '' - r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL) + r = re.compile(fr'<{nodename_patt}[\s>].*?', re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] @@ -40,9 +40,9 @@ def xmliter(obj, nodename): def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): from lxml import etree reader = _StreamReader(obj) - tag = '{%s}%s' % (namespace, nodename) if namespace else nodename + tag = f'{{{namespace}}}{nodename}'if namespace else nodename iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) - selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) + selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename) for _, node in iterable: nodetext = etree.tostring(node, encoding='unicode') node.clear() @@ -131,8 +131,7 @@ def _body_or_str(obj, unicode=True): if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) raise TypeError( - "Object %r must be %s, not %s" - % (obj, expected_types_str, type(obj).__name__) + f"Object {obj!r} must be {expected_types_str}, not {type(obj).__name__}" ) if isinstance(obj, Response): if not unicode: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 51d276097..62df7a6ab 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -143,13 +143,20 @@ def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ - "%s %s" % (name, version) + f"{name} {version}" for name, version in scrapy_components_versions() if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) + from twisted.internet import asyncioreactor + if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): + logger.debug( + "Using asyncio event loop: %s.%s", + reactor._asyncioEventloop.__module__, + reactor._asyncioEventloop.__class__.__name__, + ) class StreamLogger: @@ -176,11 +183,11 @@ class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" def __init__(self, crawler, *args, **kwargs): - super(LogCounterHandler, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.crawler = crawler def emit(self, record): - sname = 'log_count/{}'.format(record.levelname) + sname = f'log_count/{record.levelname}' self.crawler.stats.inc_value(sname) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d6966be8e..081cd33f1 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -5,6 +5,7 @@ import os import re import hashlib import warnings +from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules @@ -38,14 +39,24 @@ def arg_to_iter(arg): def load_object(path): """Load an object given its absolute object path, and return it. - object can be the import path of a class, function, variable or an - instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' + The object can be the import path of a class, function, variable or an + instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'. + + If ``path`` is not a string, but is a callable object, such as a class or + a function, then return it as is. """ + if not isinstance(path, str): + if callable(path): + return path + else: + raise TypeError("Unexpected argument type, expected string " + "or object, got: %s" % type(path)) + try: dot = path.rindex('.') except ValueError: - raise ValueError("Error loading object '%s': not a full path" % path) + raise ValueError(f"Error loading object '{path}': not a full path") module, name = path[:dot], path[dot + 1:] mod = import_module(module) @@ -53,7 +64,7 @@ def load_object(path): try: obj = getattr(mod, name) except AttributeError: - raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name)) + raise NameError(f"Module '{module}' doesn't define any object named '{name}'") return obj @@ -162,7 +173,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): instance = objcls(*args, **kwargs) method_name = '__new__' if instance is None: - raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name)) + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") return instance @@ -184,6 +195,22 @@ def set_environ(**kwargs): os.environ[k] = v +def walk_callable(node): + """Similar to ``ast.walk``, but walks only function body and skips nested + functions defined within the node. + """ + todo = deque([node]) + walked_func_def = False + while todo: + node = todo.popleft() + if isinstance(node, ast.FunctionDef): + if walked_func_def: + continue + walked_func_def = True + todo.extend(ast.iter_child_nodes(node)) + yield node + + _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) @@ -201,7 +228,7 @@ def is_generator_with_return_value(callable): if inspect.isgeneratorfunction(callable): tree = ast.parse(dedent(inspect.getsource(callable))) - for node in ast.walk(tree): + for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True return _generator_callbacks_cache[callable] @@ -217,9 +244,10 @@ def warn_on_generator_with_return_value(spider, callable): """ if is_generator_with_return_value(callable): warnings.warn( - 'The "{}.{}" method is a generator and includes a "return" statement with a ' - 'value different than None. This could lead to unexpected behaviour. Please see ' + f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' + 'a generator and includes a "return" statement with a value ' + 'different than None. This could lead to unexpected behaviour. Please see ' 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' - 'for details about the semantics of the "return" statement within generators' - .format(spider.__class__.__name__, callable.__name__), stacklevel=2, + 'for details about the semantics of the "return" statement within generators', + stacklevel=2, ) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index b8d3ebf9d..fd13d85e3 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -20,7 +20,7 @@ def inside_project(): try: import_module(scrapy_module) except ImportError as exc: - warnings.warn("Cannot import scrapy settings module %s: %s" % (scrapy_module, exc)) + warnings.warn(f"Cannot import scrapy settings module {scrapy_module}: {exc}") else: return True return bool(closest_scrapy_cfg()) @@ -90,7 +90,7 @@ def get_project_settings(): warnings.warn( 'Use of environment variables prefixed with SCRAPY_ to override ' 'settings is deprecated. The following environment variables are ' - 'currently defined: {}'.format(setting_envvar_list), + f'currently defined: {setting_envvar_list}', ScrapyDeprecationWarning ) settings.setdict(scrapy_envvars, priority='project') diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 9204977cf..5703fd4c3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -6,10 +6,12 @@ import gc import inspect import re import sys +import warnings import weakref from functools import partial, wraps from itertools import chain +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.decorators import deprecated @@ -89,7 +91,7 @@ def to_unicode(text, encoding=None, errors='strict'): return text if not isinstance(text, (bytes, str)): raise TypeError('to_unicode must receive a bytes or str ' - 'object, got %s' % type(text).__name__) + f'object, got {type(text).__name__}') if encoding is None: encoding = 'utf-8' return text.decode(encoding, errors) @@ -102,7 +104,7 @@ def to_bytes(text, encoding=None, errors='strict'): return text if not isinstance(text, str): raise TypeError('to_bytes must receive a str or bytes ' - 'object, got %s' % type(text).__name__) + f'object, got {type(text).__name__}') if encoding is None: encoding = 'utf-8' return text.encode(encoding, errors) @@ -127,6 +129,7 @@ def re_rsearch(pattern, text, chunk_size=1024): In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing the start position of the match, and the ending (regarding the entire text). """ + def _chunk_iter(): offset = len(text) while True: @@ -158,6 +161,7 @@ def memoizemethod_noargs(method): if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] + return new_method @@ -170,7 +174,7 @@ def binary_is_text(data): does not contain unprintable control characters. """ if not isinstance(data, bytes): - raise TypeError("data must be bytes, got '%s'" % type(data).__name__) + raise TypeError(f"data must be bytes, got '{type(data).__name__}'") return all(c not in _BINARYCHARS for c in data) @@ -194,7 +198,8 @@ def _getargspec_py23(func): def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): - func_args, _, _, _ = _getargspec_py23(func) + spec = inspect.getfullargspec(func) + func_args = spec.args + spec.kwonlyargs elif inspect.isclass(func): return get_func_args(func.__init__, True) elif inspect.ismethod(func): @@ -212,7 +217,7 @@ def get_func_args(func, stripself=False): else: return get_func_args(func.__call__, True) else: - raise TypeError('%s is not callable' % type(func)) + raise TypeError(f'{type(func)} is not callable') if stripself: func_args.pop(0) return func_args @@ -245,7 +250,7 @@ def get_spec(func): elif hasattr(func, '__call__'): spec = _getargspec_py23(func.__call__) else: - raise TypeError('%s is not callable' % type(func)) + raise TypeError(f'{type(func)} is not callable') defaults = spec.defaults or [] @@ -276,6 +281,7 @@ def equal_attributes(obj1, obj2, attributes): class WeakKeyCache: def __init__(self, default_factory): + warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) self.default_factory = default_factory self._weakdict = weakref.WeakKeyDictionary() @@ -316,7 +322,7 @@ def global_object_name(obj): >>> global_object_name(Request) 'scrapy.http.request.Request' """ - return "%s.%s" % (obj.__module__, obj.__name__) + return f"{obj.__module__}.{obj.__name__}" if hasattr(sys, "pypy_version_info"): diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 3c705f69b..831d29462 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -10,7 +10,7 @@ def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor if len(portrange) > 2: - raise ValueError("invalid portrange: %s" % portrange) + raise ValueError(f"invalid portrange: {portrange}") if not portrange: return reactor.listenTCP(0, factory, interface=host) if not hasattr(portrange, '__iter__'): @@ -50,13 +50,19 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) -def install_reactor(reactor_path): +def install_reactor(reactor_path, event_loop_path=None): """Installs the :mod:`~twisted.internet.reactor` with the specified - import path.""" + import path. Also installs the asyncio event loop with the specified import + path if the asyncio reactor is enabled""" reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - asyncioreactor.install(asyncio.get_event_loop()) + if event_loop_path is not None: + event_loop_class = load_object(event_loop_path) + event_loop = event_loop_class() + else: + event_loop = asyncio.new_event_loop() + asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") installer_path = module + ["install"] @@ -72,9 +78,9 @@ def verify_installed_reactor(reactor_path): from twisted.internet import reactor reactor_class = load_object(reactor_path) if not isinstance(reactor, reactor_class): - msg = "The installed reactor ({}.{}) does not match the requested one ({})".format( - reactor.__module__, reactor.__class__.__name__, reactor_path - ) + msg = ("The installed reactor " + f"({reactor.__module__}.{reactor.__class__.__name__}) does not " + f"match the requested one ({reactor_path})") raise Exception(msg) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 5ea2aafb8..d38b1bc4d 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -71,25 +71,20 @@ def request_from_dict(d, spider=None): def _find_method(obj, func): - if obj: - try: - func_self = func.__self__ - except AttributeError: # func has no __self__ - pass - else: - if func_self is obj: - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name - raise ValueError("Function %s is not a method of: %s" % (func, obj)) + # Only instance methods contain ``__func__`` + if obj and hasattr(func, '__func__'): + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name + raise ValueError(f"Function {func} is not an instance method in: {obj}") def _get_method(obj, name): @@ -97,4 +92,4 @@ def _get_method(obj, name): try: return getattr(obj, name) except AttributeError: - raise ValueError("Method %r not found in: %s" % (name, obj)) + raise ValueError(f"Method {name!r} not found in: {obj}") diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c29b619ce..99b089b6f 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -39,7 +39,7 @@ def response_status_message(status): """Return status code plus status text descriptive message """ message = http.RESPONSES.get(int(status), "Unknown Status") - return '%s %s' % (status, to_unicode(message)) + return f'{status} {to_unicode(message)}' def response_httprepr(response): @@ -69,15 +69,15 @@ def open_in_browser(response, _openfunc=webbrowser.open): body = response.body if isinstance(response, HtmlResponse): if b'' body = body.replace(b'', to_bytes(repl)) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' else: - raise TypeError("Unsupported response type: %s" % - response.__class__.__name__) + raise TypeError("Unsupported response type: " + f"{response.__class__.__name__}") fd, fname = tempfile.mkstemp(ext) os.write(fd, body) os.close(fd) - return _openfunc("file://%s" % fname) + return _openfunc(f"file://{fname}") diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index dc9604578..a73cf03c5 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -17,7 +17,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): if isinstance(o, set): return list(o) elif isinstance(o, datetime.datetime): - return o.strftime("%s %s" % (self.DATE_FORMAT, self.TIME_FORMAT)) + return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}") elif isinstance(o, datetime.date): return o.strftime(self.DATE_FORMAT) elif isinstance(o, datetime.time): @@ -29,11 +29,11 @@ class ScrapyJSONEncoder(json.JSONEncoder): elif is_item(o): return ItemAdapter(o).asdict() elif isinstance(o, Request): - return "<%s %s %s>" % (type(o).__name__, o.method, o.url) + return f"<{type(o).__name__} {o.method} {o.url}>" elif isinstance(o, Response): - return "<%s %s %s>" % (type(o).__name__, o.status, o.url) + return f"<{type(o).__name__} {o.status} {o.url}>" else: - return super(ScrapyJSONEncoder, self).default(o) + return super().default(o) class ScrapyJSONDecoder(json.JSONDecoder): diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index c3c5e329b..ea4dde882 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -50,7 +50,7 @@ def get_temp_key_info(ssl_object): key_info.append(ffi_buf_to_string(cname)) else: key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) - key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)) + key_info.append(f'{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits') return ', '.join(key_info) @@ -58,4 +58,4 @@ def get_openssl_version(): system_openssl = OpenSSL.SSL.SSLeay_version( OpenSSL.SSL.SSLEAY_VERSION ).decode('ascii', errors='replace') - return '{} ({})'.format(OpenSSL.version.__version__, system_openssl) + return f'{OpenSSL.version.__version__} ({system_openssl})' diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 96ff4b09b..f068be737 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -12,10 +12,12 @@ def render_templatefile(path, **kwargs): content = string.Template(raw).substitute(**kwargs) render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path + + if path.endswith('.tmpl'): + os.rename(path, render_path) + with open(render_path, 'wb') as fp: fp.write(content.encode('utf8')) - if path.endswith('.tmpl'): - os.remove(path) CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 7442a2f33..f54942ffb 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -79,7 +79,7 @@ def get_ftp_content_and_delete( def buffer_data(data): ftp_data.append(data) - ftp.retrbinary('RETR %s' % path, buffer_data) + ftp.retrbinary(f'RETR {path}', buffer_data) dirname, filename = split(path) ftp.cwd(dirname) ftp.delete(filename) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index a63c9a942..a54c7db95 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -23,10 +23,10 @@ class ProcessTest: def _process_finished(self, pp, cmd, check_code): if pp.exitcode and check_code: - msg = "process %s exit with code %d" % (cmd, pp.exitcode) - msg += "\n>>> stdout <<<\n%s" % pp.out + msg = f"process {cmd} exit with code {pp.exitcode}" + msg += f"\n>>> stdout <<<\n{pp.out}" msg += "\n" - msg += "\n>>> stderr <<<\n%s" % pp.err + msg += f"\n>>> stderr <<<\n{pp.err}" raise RuntimeError(msg) return pp.exitcode, pp.out, pp.err diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 66930ad2c..fce77be32 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -7,12 +7,12 @@ class SiteTest: def setUp(self): from twisted.internet import reactor - super(SiteTest, self).setUp() + super().setUp() self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - self.baseurl = "http://localhost:%d/" % self.site.getHost().port + self.baseurl = f"http://localhost:{self.site.getHost().port}/" def tearDown(self): - super(SiteTest, self).tearDown() + super().tearDown() self.site.stopListening() def url(self, path): @@ -40,5 +40,5 @@ def test_site(): if __name__ == '__main__': from twisted.internet import reactor port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - print("http://localhost:%d/" % port.getHost().port) + print(f"http://localhost:{port.getHost().port}/") reactor.run() diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index baed5c536..3e40acd69 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -41,9 +41,7 @@ def format_live_refs(ignore=NoneType): if issubclass(cls, ignore): continue oldest = min(wdict.values()) - s += "%-30s %6d oldest: %ds ago\n" % ( - cls.__name__, len(wdict), now - oldest - ) + s += f"{cls.__name__:<30} {len(wdict):6} oldest: {int(now - oldest)}s ago\n" return s diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index b23ddb459..a6a2a9e8b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,7 +22,7 @@ def url_is_from_any_domain(url, domains): if not host: return False domains = [d.lower() for d in domains] - return any((host == d) or (host.endswith('.%s' % d)) for d in domains) + return any((host == d) or (host.endswith(f'.{d}')) for d in domains) def url_is_from_spider(url, spider): @@ -153,7 +153,7 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443), ('ftp', 21)): - netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') + netloc = netloc.replace(f':{parsed_url.port}', '') return urlunparse(( parsed_url.scheme, netloc, diff --git a/setup.cfg b/setup.cfg index f8e7c0c91..8101443e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,9 +16,6 @@ ignore_errors = True [mypy-scrapy.commands] ignore_errors = True -[mypy-scrapy.commands.bench] -ignore_errors = True - [mypy-scrapy.commands.parse] ignore_errors = True @@ -28,9 +25,6 @@ ignore_errors = True [mypy-scrapy.contracts] ignore_errors = True -[mypy-scrapy.core.spidermw] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True @@ -70,15 +64,6 @@ ignore_errors = True [mypy-tests.mocks.dummydbm] ignore_errors = True -[mypy-tests.spiders] -ignore_errors = True - -[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception] -ignore_errors = True - -[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal] -ignore_errors = True - [mypy-tests.test_command_fetch] ignore_errors = True @@ -94,9 +79,6 @@ ignore_errors = True [mypy-tests.test_contracts] ignore_errors = True -[mypy-tests.test_crawler] -ignore_errors = True - [mypy-tests.test_downloader_handlers] ignore_errors = True @@ -127,50 +109,20 @@ ignore_errors = True [mypy-tests.test_pipeline_images] ignore_errors = True -[mypy-tests.test_pipelines] -ignore_errors = True - [mypy-tests.test_request_cb_kwargs] ignore_errors = True -[mypy-tests.test_request_left] -ignore_errors = True - [mypy-tests.test_scheduler] ignore_errors = True -[mypy-tests.test_signals] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.nested.spider4] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider1] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider2] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider3] -ignore_errors = True - [mypy-tests.test_spidermiddleware_httperror] ignore_errors = True -[mypy-tests.test_spidermiddleware_output_chain] -ignore_errors = True - [mypy-tests.test_spidermiddleware_referer] ignore_errors = True -[mypy-tests.test_utils_reqser] -ignore_errors = True - [mypy-tests.test_utils_serialize] ignore_errors = True -[mypy-tests.test_utils_spider] -ignore_errors = True - [mypy-tests.test_utils_url] ignore_errors = True diff --git a/setup.py b/setup.py index 58090f7a2..0c2281400 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ install_requires = [ 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', - 'lxml>=3.5.0', 'parsel>=1.5.0', 'PyDispatcher>=2.0.5', 'pyOpenSSL>=16.2.0', @@ -42,7 +41,7 @@ if has_environment_marker_platform_impl_support(): ] extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by - # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no, + # https://foss.heptapod.net/pypy/pypy/-/issues/2498, # which was fixed in Cython 0.26, released on 2017-06-19, and used to # generate the C headers of lxml release tarballs published since then, the # first of which was: @@ -83,7 +82,6 @@ setup( 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', @@ -93,7 +91,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.5.2', + python_requires='>=3.6', install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py new file mode 100644 index 000000000..1e4ada722 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop" +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 3f9738798..f545de39f 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -38,7 +38,7 @@ class LocalhostSpider(Spider): if __name__ == "__main__": with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: port = urlparse(mock_http_server.http_address).port - url = "http://not.a.real.domain:{port}/echo".format(port=port) + url = f"http://not.a.real.domain:{port}/echo" servers = [(mock_dns_server.host, mock_dns_server.port)] reactor.installResolver(createResolver(servers=servers)) diff --git a/tests/ftpserver.py b/tests/ftpserver.py new file mode 100644 index 000000000..6f0289e08 --- /dev/null +++ b/tests/ftpserver.py @@ -0,0 +1,24 @@ +from argparse import ArgumentParser + +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import FTPServer + + +def main(): + parser = ArgumentParser() + parser.add_argument('-d', '--directory') + args = parser.parse_args() + + authorizer = DummyAuthorizer() + full_permissions = 'elradfmwMT' + authorizer.add_anonymous(args.directory, perm=full_permissions) + handler = FTPHandler + handler.authorizer = authorizer + address = ('127.0.0.1', 2121) + server = FTPServer(address, handler) + server.serve_forever() + + +if __name__ == '__main__': + main() diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem index 08004feca..cdef75f99 100644 --- a/tests/keys/mitmproxy-ca.pem +++ b/tests/keys/mitmproxy-ca.pem @@ -1,32 +1,50 @@ ------BEGIN RSA PRIVATE KEY----- -MIICWwIBAAKBgQDKLbznLxS7HSWvrmGcvVS6eQvjEWD705/csvnk/WtqAPfQMJKt -auFBxzPt6RT60SHtj/2FKt2gqsiE6cNINxGN6fGYD7HtaM5HXRVPUKJaMipJwHha -QivjIZoueraY/MtlyCkpp6dmMnHEpGY7OzwMyh1eCBHQ2JYx6VEzbks9ewIDAQAB -AoGAMpS2ye/Rc+6a2xT5fskvRWe7PZe/d8E+IWz1cACmuuJ7HS7Jw3EV4esAZukF -QqrHnjOD7akHwYZ4nCgPnyWH0lLx/4TIXE5QeLPFrhKOsSLCyhlCwNVJAdcOrDol -Qh2694Dsd4gAy5o6TA02cBpqArnbAUERX46bHBZRA+ths8ECQQD6r1Ls+bTBR52w -T3rPPhYj7EsXp40MJt0pLf1kjf+EH1bxsUqnxLawwo/lLE9omU73DFnfrAflk2Ll -KUPCjjYpAkEAznchXk2ITeRcClrBNA+1Izpb5yG1qkfc79u/CEVDDOvt7RO/89Oj -58R3pKTyffoo34fBdJz8GYDsmOeiyJEjAwJANpSHrJrtlQt/tMyJQ6gT7/xZmSvc -1OF9U6L0wbj9AgpExtjAFWkKEdA6vj34iCChBb8FrmJpUb3WUWi7nReTiQJAFyIT -9Av93LRcd7CJezrTUdolF/WX9DdPEvTtJ5ETHSyGIQ0Yccph0AMcYK82mFTiJYGB -dH5uZLEkUVGK1KwmXwJAGWLdYiQyQitRWdoURcLb4OZ2gF3+7PASgFilI8YuoYhn -Rl2Va3UtErPKJeMg2dTH18PuXykQMsQR1+rPxf1WSA== ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB +/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT +NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ +u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH +JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB +eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls +90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ +epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8 +VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn +v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT +I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU +oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au +/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG +gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn +YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf +dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5 +9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC +QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu +bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW +CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR +OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu +PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK +eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx +BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/ +B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy +HzPh2taMnCA0h1fNLFFBkw88 +-----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICnzCCAgigAwIBAgIGDI2K/EOjMA0GCSqGSIb3DQEBBQUAMCgxEjAQBgNVBAMT -CW1pdG1wcm94eTESMBAGA1UEChMJbWl0bXByb3h5MB4XDTEzMDkyNjE0MzYxMVoX -DTE1MDkxNjE0MzYxMVowKDESMBAGA1UEAxMJbWl0bXByb3h5MRIwEAYDVQQKEwlt -aXRtcHJveHkwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMotvOcvFLsdJa+u -YZy9VLp5C+MRYPvTn9yy+eT9a2oA99Awkq1q4UHHM+3pFPrRIe2P/YUq3aCqyITp -w0g3EY3p8ZgPse1ozkddFU9QoloyKknAeFpCK+Mhmi56tpj8y2XIKSmnp2YyccSk -Zjs7PAzKHV4IEdDYljHpUTNuSz17AgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB -/zAUBglghkgBhvhCAQEBAf8EBAMCAgQwewYDVR0lAQH/BHEwbwYIKwYBBQUHAwEG -CCsGAQUFBwMCBggrBgEFBQcDBAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQB -gjcCARYGCisGAQQBgjcKAwEGCisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG -+EIEATALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFJBEfawVwhEHHW6rS8nvZFlJ582n -MA0GCSqGSIb3DQEBBQUAA4GBAHGl28Ip2CWS/MibCaFztLDxGiMBT4MW2yI2hf3D -y9g1o7ra/fSEFdIc849xXyCsGWSkMsbDML272rCH4K73MUBxxkJm46AIyRVH1z2Z -e96u4py1wNT8cznY15phr8pn36snlaHaYa+JcwGINMdSOk1VPHv6gqSC/vgUCgF1 -n95u +MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM +CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX +DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt +aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW +ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7 +ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg +z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI +ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE +6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/ +0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC +AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD +BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG +CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC +AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA +A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd +kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe +gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY +1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj +envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj +wiKjdLjCmJgjGAQjRw5VYJ8JI0XO -----END CERTIFICATE----- diff --git a/tests/mockserver.py b/tests/mockserver.py index 1f40473ba..ab9aec6a6 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -3,7 +3,10 @@ import json import os import random import sys +from pathlib import Path +from shutil import rmtree from subprocess import Popen, PIPE +from tempfile import mkdtemp from urllib.parse import urlencode from OpenSSL import SSL @@ -70,7 +73,7 @@ class Follow(LeafResource): for nl in nlist: args[b"n"] = [to_bytes(str(nl))] argstr = urlencode(args, doseq=True) - s += "follow %d
    " % (argstr, nl) + s += f"follow {nl}
    " s += """""" request.write(to_bytes(s)) request.finish() @@ -88,7 +91,7 @@ class Delay(LeafResource): return NOT_DONE_YET def _delayedRender(self, request, n): - request.write(to_bytes("Response delayed for %0.3f seconds\n" % n)) + request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) request.finish() @@ -256,6 +259,29 @@ class MockDNSServer: self.proc.communicate() +class MockFTPServer: + """Creates an FTP server on port 2121 with a default passwordless user + (anonymous) and a temporary root path that you can read from the + :attr:`path` attribute.""" + + def __enter__(self): + self.path = Path(mkdtemp()) + self.proc = Popen([sys.executable, '-u', '-m', 'tests.ftpserver', '-d', str(self.path)], + stderr=PIPE, env=get_testenv()) + for line in self.proc.stderr: + if b'starting FTP server' in line: + break + return self + + def __exit__(self, exc_type, exc_value, traceback): + rmtree(str(self.path)) + self.proc.kill() + self.proc.communicate() + + def url(self, path): + return 'ftp://127.0.0.1:2121/' + path + + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( os.path.join(os.path.dirname(__file__), keyfile), @@ -263,8 +289,8 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c ) if cipher_string: ctx = factory.getContext() - # disabling TLS1.2+ because it unconditionally enables some strong ciphers - ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3) + # disabling TLS1.3 because it unconditionally enables some strong ciphers + ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3) ctx.set_cipher_list(to_bytes(cipher_string)) return factory @@ -284,8 +310,8 @@ if __name__ == "__main__": def print_listening(): httpHost = httpPort.getHost() httpsHost = httpsPort.getHost() - httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port) - httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port) + httpAddress = f'http://{httpHost.host}:{httpHost.port}' + httpsAddress = f'https://{httpsHost.host}:{httpsHost.port}' print(httpAddress) print(httpsAddress) @@ -297,7 +323,7 @@ if __name__ == "__main__": def print_listening(): host = listener.getHost() - print("%s:%s" % (host.host, host.port)) + print(f"{host.host}:{host.port}") reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py deleted file mode 100644 index 162a53760..000000000 --- a/tests/py36/_test_crawl.py +++ /dev/null @@ -1,57 +0,0 @@ -import asyncio - -from scrapy import Request -from tests.spiders import SimpleSpider - - -class AsyncDefAsyncioGenSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen' - - async def parse(self, response): - await asyncio.sleep(0.2) - yield {'foo': 42} - self.logger.info("Got response %d" % response.status) - - -class AsyncDefAsyncioGenLoopSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen_loop' - - async def parse(self, response): - for i in range(10): - await asyncio.sleep(0.1) - yield {'foo': i} - self.logger.info("Got response %d" % response.status) - - -class AsyncDefAsyncioGenComplexSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen_complex' - initial_reqs = 4 - following_reqs = 3 - depth = 2 - - def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), - meta={'index': index}, - dont_filter=True, - callback=cb) - - def start_requests(self): - for i in range(1, self.initial_reqs + 1): - yield self._get_req(i) - - async def parse(self, response): - index = response.meta['index'] - yield {'index': index} - if index < 10 ** self.depth: - for new_index in range(10 * index, 10 * index + self.following_reqs): - yield self._get_req(new_index) - yield self._get_req(index, cb=self.parse2) - await asyncio.sleep(0.1) - yield {'index': index + 5} - - async def parse2(self, response): - await asyncio.sleep(0.1) - yield {'index2': response.meta['index']} diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 00c56084d..2247ed917 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,9 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy; python_version >= '3.6' -mitmproxy<4.0.0; python_version < '3.6' +mitmproxy; python_version >= '3.7' +mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' +pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines @@ -11,6 +12,7 @@ pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures +uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython diff --git a/tests/spiders.py b/tests/spiders.py index 3eb681819..106392ea6 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queue class MockServerSpider(Spider): def __init__(self, mockserver=None, *args, **kwargs): - super(MockServerSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.mockserver = mockserver @@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider): name = 'meta' def __init__(self, *args, **kwargs): - super(MetaSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.meta = {} def closed(self, reason): @@ -41,11 +41,11 @@ class FollowAllSpider(MetaSpider): link_extractor = LinkExtractor() def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs): - super(FollowAllSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1)) + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") self.start_urls = [url] def parse(self, response): @@ -60,14 +60,14 @@ class DelaySpider(MetaSpider): name = 'delay' def __init__(self, n=1, b=0, *args, **kwargs): - super(DelaySpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.n = n self.b = b self.t1 = self.t2 = self.t2_err = 0 def start_requests(self): self.t1 = time.time() - url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b)) + url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) def parse(self, response): @@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider): name = 'simple' def __init__(self, url="http://localhost:8998", *args, **kwargs): - super(SimpleSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [url] def parse(self, response): @@ -148,12 +148,65 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): return reqs +class AsyncDefAsyncioGenSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen' + + async def parse(self, response): + await asyncio.sleep(0.2) + yield {'foo': 42} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenLoopSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_loop' + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {'foo': i} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenComplexSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_complex' + initial_reqs = 4 + following_reqs = 3 + depth = 2 + + def _get_req(self, index, cb=None): + return Request(self.mockserver.url(f"/status?n=200&request={index}"), + meta={'index': index}, + dont_filter=True, + callback=cb) + + def start_requests(self): + for i in range(1, self.initial_reqs + 1): + yield self._get_req(i) + + async def parse(self, response): + index = response.meta['index'] + yield {'index': index} + if index < 10 ** self.depth: + for new_index in range(10 * index, 10 * index + self.following_reqs): + yield self._get_req(new_index) + yield self._get_req(index, cb=self.parse2) + await asyncio.sleep(0.1) + yield {'index': index + 5} + + async def parse2(self, response): + await asyncio.sleep(0.1) + yield {'index2': response.meta['index']} + + class ItemSpider(FollowAllSpider): name = 'item' def parse(self, response): - for request in super(ItemSpider, self).parse(response): + for request in super().parse(response): yield request yield Item() yield {} @@ -172,7 +225,7 @@ class ErrorSpider(FollowAllSpider): raise self.exception_cls('Expected exception') def parse(self, response): - for request in super(ErrorSpider, self).parse(response): + for request in super().parse(response): yield request self.raise_exception() @@ -183,7 +236,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): fail_yielding = False def __init__(self, *a, **kw): - super(BrokenStartRequestsSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self.seedsseen = [] def start_requests(self): @@ -192,7 +245,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1) + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 @@ -201,7 +254,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): def parse(self, response): self.seedsseen.append(response.meta.get('seed')) - for req in super(BrokenStartRequestsSpider, self).parse(response): + for req in super().parse(response): yield req @@ -239,11 +292,11 @@ class DuplicateStartRequestsSpider(MockServerSpider): def start_requests(self): for i in range(0, self.distinct_urls): for j in range(0, self.dupe_factor): - url = self.mockserver.url("/echo?headers=1&body=test%d" % i) + url = self.mockserver.url(f"/echo?headers=1&body=test{i}") yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): - super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.visited = 0 def parse(self, response): @@ -255,7 +308,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): A CrawlSpider which overrides the 'parse' method """ name = 'crawl_spider_with_parse_method' - custom_settings = { + custom_settings: dict = { 'RETRY_HTTP_CODES': [], # no need to retry } rules = ( diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index da99a6be8..591075a98 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase): 'EXTENSIONS=' + json.dumps(EXTENSIONS)) # XXX: There's gotta be a smarter way to do this... self.assertNotIn("...", settingsstr) - for char in ("'", "<", ">", 'u"'): + for char in ("'", "<", ">"): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index 6504b4d2c..005e45214 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -4,7 +4,7 @@ class TestExtension: def __init__(self, settings): - settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started')) + settings.set('TEST1', f"{settings['TEST1']} + started") @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_command_check.py b/tests/test_command_check.py new file mode 100644 index 000000000..34f5e59dd --- /dev/null +++ b/tests/test_command_check.py @@ -0,0 +1,97 @@ +from os.path import join, abspath + +from tests.test_commands import CommandTest + + +class CheckCommandTest(CommandTest): + + command = 'check' + + def setUp(self): + super(CheckCommandTest, self).setUp() + self.spider_name = 'check_spider' + self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py')) + + def _write_contract(self, contracts, parse_def): + with open(self.spider, 'w') as file: + file.write(f""" +import scrapy + +class CheckSpider(scrapy.Spider): + name = '{self.spider_name}' + start_urls = ['http://example.com'] + + def parse(self, response, **cb_kwargs): + \"\"\" + @url http://example.com + {contracts} + \"\"\" + {parse_def} + """) + + def _test_contract(self, contracts='', parse_def='pass'): + self._write_contract(contracts, parse_def) + p, out, err = self.proc('check') + self.assertNotIn('F', out) + self.assertIn('OK', err) + self.assertEqual(p.returncode, 0) + + def test_check_returns_requests_contract(self): + contracts = """ + @returns requests 1 + """ + parse_def = """ + yield scrapy.Request(url='http://next-url.com') + """ + self._test_contract(contracts, parse_def) + + def test_check_returns_items_contract(self): + contracts = """ + @returns items 1 + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + """ + self._test_contract(contracts, parse_def) + + def test_check_cb_kwargs_contract(self): + contracts = """ + @cb_kwargs {"arg1": "val1", "arg2": "val2"} + """ + parse_def = """ + if len(cb_kwargs.items()) == 0: + raise Exception("Callback args not set") + """ + self._test_contract(contracts, parse_def) + + def test_check_scrapes_contract(self): + contracts = """ + @scrapes key1 key2 + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + """ + self._test_contract(contracts, parse_def) + + def test_check_all_default_contracts(self): + contracts = """ + @returns items 1 + @returns requests 1 + @scrapes key1 key2 + @cb_kwargs {"arg1": "val1", "arg2": "val2"} + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + yield scrapy.Request(url='http://next-url.com') + if len(cb_kwargs.items()) == 0: + raise Exception("Callback args not set") + """ + self._test_contract(contracts, parse_def) + + def test_SCRAPY_CHECK_set(self): + parse_def = """ + import os + if not os.environ.get('SCRAPY_CHECK'): + raise Exception('SCRAPY_CHECK not set') + """ + self._test_contract(parse_def=parse_def) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 5754a5478..ed3848d88 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -17,18 +17,18 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' def setUp(self): - super(ParseCommandTest, self).setUp() + super().setUp() self.spider_name = 'parse_spider' fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) with open(fname, 'w') as f: - f.write(""" + f.write(f""" import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule class MySpider(scrapy.Spider): - name = '{0}' + name = '{self.spider_name}' def parse(self, response): if getattr(self, 'test_arg', None): @@ -58,7 +58,7 @@ class MySpider(scrapy.Spider): self.logger.debug('It Does Not Work :(') class MyGoodCrawlSpider(CrawlSpider): - name = 'goodcrawl{0}' + name = 'goodcrawl{self.spider_name}' rules = ( Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), @@ -74,7 +74,7 @@ class MyGoodCrawlSpider(CrawlSpider): class MyBadCrawlSpider(CrawlSpider): '''Spider which doesn't define a parse_item callback while using it in a rule.''' - name = 'badcrawl{0}' + name = 'badcrawl{self.spider_name}' rules = ( Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), @@ -82,7 +82,7 @@ class MyBadCrawlSpider(CrawlSpider): def parse(self, response): return [scrapy.Item(), dict(foo='bar')] -""".format(self.spider_name)) +""") fname = abspath(join(self.proj_mod_path, 'pipelines.py')) with open(fname, 'w') as f: @@ -99,9 +99,9 @@ class MyPipeline: fname = abspath(join(self.proj_mod_path, 'settings.py')) with open(fname, 'a') as f: - f.write(""" -ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} -""" % self.project_name) + f.write(f""" +ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} +""") @defer.inlineCallbacks def test_spider_arguments(self): diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 66c293c00..16c9559b5 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -65,8 +65,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): def test_fetch_redirect_follow_302(self): """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" url = self.url('/redirect-no-meta-refresh') - code = "fetch('{0}')" - errcode, out, errout = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}')" + errcode, out, errout = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) assert b'Redirecting (302)' in errout assert b'Crawled (200)' in errout @@ -75,23 +75,23 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): def test_fetch_redirect_not_follow_302(self): """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" url = self.url('/redirect-no-meta-refresh') - code = "fetch('{0}', redirect=False)" - errcode, out, errout = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}', redirect=False)" + errcode, out, errout = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) assert b'Crawled (302)' in errout @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') - code = "fetch('{0}') or fetch(response.request.replace(method='POST'))" - errcode, out, _ = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" + errcode, out, _ = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) @defer.inlineCallbacks def test_scrapy_import(self): url = self.url('/text') - code = "fetch(scrapy.Request('{0}'))" - errcode, out, _ = yield self.execute(['-c', code.format(url)]) + code = f"fetch(scrapy.Request('{url}'))" + errcode, out, _ = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) @defer.inlineCallbacks diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 99c01c2b7..00d998388 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -16,7 +16,7 @@ class VersionTest(ProcessTest, unittest.TestCase): _, out, _ = yield self.execute([]) self.assertEqual( out.strip().decode(encoding), - "Scrapy %s" % scrapy.__version__, + f"Scrapy {scrapy.__version__}", ) @defer.inlineCallbacks diff --git a/tests/test_commands.py b/tests/test_commands.py index 42091ab00..2899e5f24 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -8,7 +8,7 @@ import sys import tempfile from contextlib import contextmanager from itertools import chain -from os.path import exists, join, abspath +from os.path import exists, join, abspath, getmtime from pathlib import Path from shutil import rmtree, copytree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION @@ -16,6 +16,7 @@ from tempfile import mkdtemp from threading import Timer from unittest import skipIf +from pytest import mark from twisted.trial import unittest import scrapy @@ -41,7 +42,7 @@ class CommandSettings(unittest.TestCase): def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' - opts, args = self.parser.parse_args(args=['-s', 'FEEDS={}'.format(feeds_json), 'spider.py']) + opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) @@ -74,6 +75,7 @@ class ProjectTest(unittest.TestCase): def kill_proc(): p.kill() + p.communicate() assert False, 'Command took too much time to complete' timer = Timer(15, kill_proc) @@ -126,9 +128,13 @@ class StartprojectTest(ProjectTest): def get_permissions_dict(path, renamings=None, ignore=None): + + def get_permissions(path): + return oct(os.stat(path).st_mode) + renamings = renamings or tuple() permissions_dict = { - '.': os.stat(path).st_mode, + '.': get_permissions(path), } for root, dirs, files in os.walk(path): nodes = list(chain(dirs, files)) @@ -143,15 +149,17 @@ def get_permissions_dict(path, renamings=None, ignore=None): search_string, replacement ) - permissions = os.stat(absolute_path).st_mode + permissions = get_permissions(absolute_path) permissions_dict[relative_path] = permissions return permissions_dict class StartprojectTemplatesTest(ProjectTest): + maxDiff = None + def setUp(self): - super(StartprojectTemplatesTest, self).setUp() + super().setUp() self.tmpl = join(self.temp_path, 'templates') self.tmpl_proj = join(self.tmpl, 'project') @@ -161,10 +169,10 @@ class StartprojectTemplatesTest(ProjectTest): pass assert exists(join(self.tmpl_proj, 'root_template')) - args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] + args = ['--set', f'TEMPLATES_DIR={self.tmpl}'] p, out, err = self.proc('startproject', self.project_name, *args) - self.assertIn("New Scrapy project '%s', using template directory" - % self.project_name, out) + self.assertIn(f"New Scrapy project '{self.project_name}', " + "using template directory", out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) @@ -245,7 +253,7 @@ class StartprojectTemplatesTest(ProjectTest): 'startproject', project_name, '--set', - 'TEMPLATES_DIR={}'.format(read_only_templates_dir), + f'TEMPLATES_DIR={read_only_templates_dir}', ), cwd=destination, env=self.env, @@ -291,7 +299,7 @@ class StartprojectTemplatesTest(ProjectTest): path.mkdir(mode=permissions) else: path.touch(mode=permissions) - expected_permissions[node] = path.stat().st_mode + expected_permissions[node] = oct(path.stat().st_mode) process = subprocess.Popen( ( @@ -311,14 +319,61 @@ class StartprojectTemplatesTest(ProjectTest): self.assertEqual(actual_permissions, expected_permissions) + def test_startproject_permissions_umask_022(self): + """Check that generated files have the right permissions when the + system uses a umask value that causes new files to have different + permissions than those from the template folder.""" + @contextmanager + def umask(new_mask): + cur_mask = os.umask(new_mask) + yield + os.umask(cur_mask) + + scrapy_path = scrapy.__path__[0] + project_template = os.path.join( + scrapy_path, + 'templates', + 'project' + ) + project_name = 'umaskproject' + renamings = ( + ('module', project_name), + ('.tmpl', ''), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + with umask(0o002): + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'startproject', + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = os.path.join(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + self.assertEqual(actual_permissions, expected_permissions) + class CommandTest(ProjectTest): def setUp(self): - super(CommandTest, self).setUp() + super().setUp() self.call('startproject', self.project_name) self.cwd = join(self.temp_path, self.project_name) - self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name + self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings' class GenspiderCommandTest(CommandTest): @@ -332,13 +387,16 @@ class GenspiderCommandTest(CommandTest): assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) def test_template(self, tplname='crawl'): - args = ['--template=%s' % tplname] if tplname else [] + args = [f'--template={tplname}'] if tplname else [] spname = 'test_spider' p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) + self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) + modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn("Spider %r already exists in module" % spname, out) + self.assertIn(f"Spider {spname!r} already exists in module", out) + modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + self.assertEqual(modify_time_after, modify_time_before) def test_template_basic(self): self.test_template('basic') @@ -358,7 +416,41 @@ class GenspiderCommandTest(CommandTest): def test_same_name_as_project(self): self.assertEqual(2, self.call('genspider', self.project_name)) - assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name)) + assert not exists(join(self.proj_mod_path, 'spiders', f'{self.project_name}.py')) + + def test_same_filename_as_existing_spider(self, force=False): + file_name = 'example' + file_path = join(self.proj_mod_path, 'spiders', f'{file_name}.py') + self.assertEqual(0, self.call('genspider', file_name, 'example.com')) + assert exists(file_path) + + # change name of spider but not its file name + with open(file_path, 'r+') as spider_file: + file_data = spider_file.read() + file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") + spider_file.seek(0) + spider_file.write(file_data) + spider_file.truncate() + modify_time_before = getmtime(file_path) + file_contents_before = file_data + + if force: + p, out, err = self.proc('genspider', '--force', file_name, 'example.com') + self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) + modify_time_after = getmtime(file_path) + self.assertNotEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertNotEqual(file_contents_after, file_contents_before) + else: + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn(f"{file_path} already exists", out) + modify_time_after = getmtime(file_path) + self.assertEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertEqual(file_contents_after, file_contents_before) + + def test_same_filename_as_existing_spider_force(self): + self.test_same_filename_as_existing_spider(force=True) class GenspiderStandaloneCommandTest(ProjectTest): @@ -367,6 +459,34 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.call('genspider', 'example', 'example.com') assert exists(join(self.temp_path, 'example.py')) + def test_same_name_as_existing_file(self, force=False): + file_name = 'example' + file_path = join(self.temp_path, file_name + '.py') + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) + assert exists(file_path) + modify_time_before = getmtime(file_path) + file_contents_before = open(file_path, 'r').read() + + if force: + # use different template to ensure contents were changed + p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') + self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) + modify_time_after = getmtime(file_path) + self.assertNotEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertNotEqual(file_contents_after, file_contents_before) + else: + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn(f"{join(self.temp_path, file_name + '.py')} already exists", out) + modify_time_after = getmtime(file_path) + self.assertEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertEqual(file_contents_after, file_contents_before) + + def test_same_name_as_existing_file_force(self): + self.test_same_name_as_existing_file(force=True) + class MiscCommandsTest(CommandTest): @@ -504,6 +624,77 @@ class BadSpider(scrapy.Spider): log = self.get_log(self.debug_log_spider, args=[]) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + def test_custom_asyncio_loop_enabled_true(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', + 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor', + '-s', + 'ASYNCIO_EVENT_LOOP=uvloop.Loop', + ]) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_custom_asyncio_loop_enabled_false(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' + ]) + import asyncio + loop = asyncio.new_event_loop() + self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log) + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return [] +""" + args = ['-o', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log) + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return [] +""" + args = ['-O', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + return [] +""" + args = ['-o', 'example1.json', '-O', 'example2.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) + class BenchCommandTest(CommandTest): @@ -512,3 +703,79 @@ class BenchCommandTest(CommandTest): '-s', 'CLOSESPIDER_TIMEOUT=0.01') self.assertIn('INFO: Crawled', log) self.assertNotIn('Unhandled Error', log) + + +class CrawlCommandTest(CommandTest): + + def crawl(self, code, args=()): + fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) + with open(fname, 'w') as f: + f.write(code) + return self.proc('crawl', 'myspider', *args) + + def get_log(self, code, args=()): + _, _, stderr = self.crawl(code, args=args) + return stderr + + def test_no_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('It works!') + return [] +""" + log = self.get_log(spider_code) + self.assertIn("[myspider] DEBUG: It works!", log) + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return [] +""" + args = ['-o', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log) + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return [] +""" + args = ['-O', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + return [] +""" + args = ['-o', 'example1.json', '-O', 'example2.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 99120b128..d0f4a68c2 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase): name = 'test_same_url' def __init__(self, *args, **kwargs): - super(TestSameUrlSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.visited = 0 def start_requests(s): @@ -393,7 +393,7 @@ class ContractsManagerTest(unittest.TestCase): return TestItem() with MockServer() as mockserver: - contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) + contract_doc = f'@url {mockserver.url("/status?n=200")}' TestSameUrlSpider.parse_first.__doc__ = contract_doc TestSameUrlSpider.parse_second.__doc__ = contract_doc diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py new file mode 100644 index 000000000..113ea8f19 --- /dev/null +++ b/tests/test_core_downloader.py @@ -0,0 +1,10 @@ +from twisted.trial import unittest + +from scrapy.core.downloader import Slot + + +class SlotTest(unittest.TestCase): + + def test_repr(self): + slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) + self.assertEqual(repr(slot), 'Slot(concurrency=8, delay=0.10, randomize_delay=True)') diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 642c24651..1083c1678 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,5 @@ import json import logging -import sys from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse @@ -20,6 +19,9 @@ from scrapy.http.response import Response from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import ( + AsyncDefAsyncioGenComplexSpider, + AsyncDefAsyncioGenLoopSpider, + AsyncDefAsyncioGenSpider, AsyncDefAsyncioReqsReturnSpider, AsyncDefAsyncioReturnSingleElementSpider, AsyncDefAsyncioReturnSpider, @@ -79,7 +81,7 @@ class CrawlTestCase(TestCase): total_time = times[-1] - times[0] average = total_time / (len(times) - 1) self.assertTrue(average > delay * tolerance, - "download delay too small: %s" % average) + f"download delay too small: {average}") # Ensure that the same test parameters would cause a failure if no # download delay is set. Otherwise, it means we are using a combination @@ -204,7 +206,7 @@ with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) + yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver) self.assertEqual(str(log).count("Got response 200"), 1) @defer.inlineCallbacks @@ -333,6 +335,19 @@ class CrawlSpiderTestCase(TestCase): def tearDown(self): self.mockserver.__exit__(None, None, None) + @defer.inlineCallbacks + def _run_spider(self, spider_cls): + items = [] + + def _on_item_scraped(item): + items.append(item) + + crawler = self.runner.create_crawler(spider_cls) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + return log, items, crawler.stats + @defer.inlineCallbacks def test_crawlspider_with_parse(self): self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver) @@ -377,15 +392,7 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) self.assertIn("Got response 200", str(log)) self.assertIn({'id': 1}, items) self.assertIn({'id': 2}, items) @@ -405,52 +412,29 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("Got response 200", str(log)) self.assertIn({"foo": 42}, items) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): - from tests.py36._test_crawl import AsyncDefAsyncioGenSpider - crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) self.assertIn("Got response 200", str(log)) - itemcount = crawler.stats.get_value('item_scraped_count') + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 1) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider - crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) self.assertIn("Got response 200", str(log)) - itemcount = crawler.stats.get_value('item_scraped_count') + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 10) for i in range(10): self.assertIn({'foo': i}, items) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider - crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) - itemcount = crawler.stats.get_value('item_scraped_count') + _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 156) # some random items for i in [1, 4, 21, 22, 207, 311]: @@ -461,11 +445,9 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): - crawler = self.runner.create_crawler(AsyncDefAsyncioReqsReturnSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): - self.assertIn("Got response 200, req_id %d" % req_id, str(log)) + self.assertIn(f"Got response 200, req_id {req_id}", str(log)) @defer.inlineCallbacks def test_response_ssl_certificate_none(self): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 1a4cfe813..85035a220 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -142,7 +142,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): def test_spider_manager_verify_interface(self): settings = Settings({ - 'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface' + 'SPIDER_LOADER_CLASS': SpiderLoaderWithWrongInterface, }) with warnings.catch_warnings(record=True) as w: self.assertRaises(AttributeError, CrawlerRunner, settings) @@ -345,6 +345,14 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + def test_custom_loop_asyncio(self): + log = self.run_script("asyncio_custom_loop.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 51deb20f4..0a374c161 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -61,7 +61,7 @@ class OffDH: class LoadTestCase(unittest.TestCase): def test_enabled_handler(self): - handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'} + handlers = {'scheme': DummyDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) @@ -69,7 +69,7 @@ class LoadTestCase(unittest.TestCase): self.assertNotIn('scheme', dh._notconfigured) def test_not_configured_handler(self): - handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'} + handlers = {'scheme': OffDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) @@ -87,7 +87,7 @@ class LoadTestCase(unittest.TestCase): self.assertIn('scheme', dh._notconfigured) def test_lazy_handlers(self): - handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'} + handlers = {'scheme': DummyLazyDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) @@ -121,7 +121,7 @@ class FileTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_non_existent(self): - request = Request('file://%s' % self.mktemp()) + request = Request(f'file://{self.mktemp()}') d = self.download_request(request, Spider('foo')) return self.assertFailure(d, IOError) @@ -249,7 +249,7 @@ class HttpTestCase(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) + return f"{self.scheme}://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL('file')) @@ -300,7 +300,7 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): self.assertEqual( - response.body, to_bytes('%s:%d' % (self.host, self.portno))) + response.body, to_bytes(f'{self.host}:{self.portno}')) self.assertEqual(request.headers, {}) request = Request(self.getURL('host')) @@ -410,7 +410,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('largechunkedfile')) def check(logger): - logger.error.assert_called_once_with(mock.ANY, mock.ANY) + logger.warning.assert_called_once_with(mock.ANY, mock.ANY) d = self.download_request(request, Spider('foo', download_maxsize=1500)) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) @@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" def setUp(self): - super(Https11InvalidDNSId, self).setUp() + super().setUp() self.host = '127.0.0.1' @@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase): 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' ) - super(Https11InvalidDNSPattern, self).setUp() + super().setUp() class Https11CustomCiphers(unittest.TestCase): @@ -583,7 +583,7 @@ class Https11CustomCiphers(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) + return f"{self.scheme}://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL('file')) @@ -678,7 +678,7 @@ class HttpProxyTestCase(unittest.TestCase): yield self.download_handler.close() def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) + return f"http://127.0.0.1:{self.portno}/{path}" def test_download_with_proxy(self): def _test(response): @@ -696,7 +696,7 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, b'https://example.com') - http_proxy = '%s?noconnect' % self.getURL('') + http_proxy = f'{self.getURL("")}?noconnect' request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): @@ -977,7 +977,7 @@ class BaseFTPTestCase(unittest.TestCase): return deferred def test_ftp_download_success(self): - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=self.req_meta) d = self.download_handler.download_request(request, None) @@ -989,7 +989,7 @@ class BaseFTPTestCase(unittest.TestCase): def test_ftp_download_path_with_spaces(self): request = Request( - url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum, + url=f"ftp://127.0.0.1:{self.portNum}/file with spaces.txt", meta=self.req_meta ) d = self.download_handler.download_request(request, None) @@ -1001,7 +1001,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): - request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt", meta=self.req_meta) d = self.download_handler.download_request(request, None) @@ -1015,7 +1015,7 @@ class BaseFTPTestCase(unittest.TestCase): os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) @@ -1037,7 +1037,7 @@ class FTPTestCase(BaseFTPTestCase): meta = dict(self.req_meta) meta.update({"ftp_password": 'invalid'}) - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) @@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "iso-8859-7") @@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase): def test_text_charset(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(response.body, b'\xbe\xd3\xbe') self.assertEqual(response.encoding, "iso-8859-7") @@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase): def test_mediatype_parameters(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "utf-8") diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index a9190c62b..79f24c8a1 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -84,7 +84,7 @@ class DefaultsTest(ManagerTestCase): }) ret = self._download(request=req, response=resp) self.assertTrue(isinstance(ret, Request), - "Not redirected: {0!r}".format(ret)) + f"Not redirected: {ret!r}") self.assertEqual(to_bytes(ret.url), resp.headers['Location'], "Not redirected to location header") diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 9ccc2110b..a3de307ee 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase): def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes - req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')}) + req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') # 2) Non UTF8-encoded bytes - req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')}) + req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')}) assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') - # 3) Unicode string - req3 = Request('http://example.org', cookies={'a': u'á'}) + # 3) String + req3 = Request('http://example.org', cookies={'a': 'á'}) assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes - req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')}) + req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') # 2) Non UTF8-encoded bytes - req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')}) + req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')}) assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') - # 3) Unicode string - req3 = Request('http://example.org', headers={'Cookie': u'a=á'}) + # 3) String + req3 = Request('http://example.org', headers={'Cookie': 'a=á'}) assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') @@ -322,6 +322,9 @@ class CookiesMiddlewareTest(TestCase): cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}] req2 = Request('http://example.org/2', cookies=cookies2) assert self.mw.process_request(req2, self.spider) is None + cookies3 = [{'name': 'foo', 'value': None}, {'name': 'key', 'value': ''}] + req3 = Request('http://example.org/3', cookies=cookies3) + assert self.mw.process_request(req3, self.spider) is None lc.check( ("scrapy.downloadermiddlewares.cookies", "WARNING", @@ -331,6 +334,11 @@ class CookiesMiddlewareTest(TestCase): "WARNING", "Invalid cookie found in request :" " {'name': 'foo'} ('value' is missing)"), + ("scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request :" + " {'name': 'foo', 'value': None} ('value' is missing)"), ) self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') + self.assertCookieValEqual(req3.headers['Cookie'], 'key=') diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index dbae4d3ae..b2b5ce77d 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -28,7 +28,7 @@ class DecompressionMiddlewareTest(TestCase): for fmt in self.test_formats: rsp = self.test_responses[fmt] new = self.mw.process_response(None, rsp, self.spider) - error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__) + error_msg = f'Failed {fmt}, response type {type(new).__name__}' assert isinstance(new, XmlResponse), error_msg assert_samelines(self, new.body, self.uncompressed_body, fmt) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9b77c97a8..0c6dcf2aa 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module) - return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings) + return super()._get_settings(**new_settings) def test_custom_dbm_module_loaded(self): # make sure our dbm module has been loaded @@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_GZIP', True) - return super(FilesystemStorageTest, self)._get_settings(**new_settings) + return super()._get_settings(**new_settings) class DummyPolicyTest(_BaseTest): @@ -324,7 +324,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (shouldcache, status, headers) in enumerate(responses): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) res1 = self._process_requestresponse(mw, req0, res0) res304 = res0.replace(status=304) @@ -343,7 +343,7 @@ class RFC2616PolicyTest(DefaultStorageTest): with self._middleware(HTTPCACHE_ALWAYS_STORE=True) as mw: for idx, (_, status, headers) in enumerate(responses): shouldcache = 'no-store' not in headers.get('Cache-Control', '') and status != 304 - req0 = Request('http://example2-%d.com' % idx) + req0 = Request(f'http://example2-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) res1 = self._process_requestresponse(mw, req0, res0) res304 = res0.replace(status=304) @@ -386,7 +386,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) # cache fresh response res1 = self._process_requestresponse(mw, req0, res0) @@ -423,7 +423,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0a = Response(req0.url, status=status, headers=headers) # cache expired response res1 = self._process_requestresponse(mw, req0, res0a) @@ -490,7 +490,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware(HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS=['no-cache', 'no-store']) as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) # cache fresh response res1 = self._process_requestresponse(mw, req0, res0) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 9841d7a76..351631eb8 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase): def test_proxy_auth_encoding(self): # utf-8 encoding - os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128' + os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128' mw = HttpProxyMiddleware(auth_encoding='utf-8') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None @@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase): self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') # proxy from request.meta - req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') @@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase): self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') # proxy from request.meta, latin-1 encoding - req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 919dbed23..816ac1440 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -22,7 +22,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_redirect_3xx_permanent(self): def _test(method, status=301): - url = 'http://www.example.com/{}'.format(status) + url = f'http://www.example.com/{status}' url2 = 'http://www.example.com/redirected' req = Request(url, method=method) rsp = Response(url, headers={'Location': url2}, status=status) @@ -79,7 +79,7 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.method, 'GET') assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" - assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" # response without Location header but with status code is 3XX should be ignored del rsp.headers['Location'] @@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_latin1_location(self): req = Request('http://scrapytest.org/first') - latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1 + latin1_location = '/ação'.encode('latin1') # HTTP historically supports latin1 resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' @@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_utf8_location(self): req = Request('http://scrapytest.org/first') - utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding + utf8_location = '/ação'.encode('utf-8') # header using UTF-8 encoding resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' @@ -207,8 +207,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.mw = MetaRefreshMiddleware.from_crawler(crawler) def _body(self, interval=5, url='http://example.org/newpage'): - html = u"""""" - return html.format(interval, url).encode('utf-8') + html = f"""""" + return html.encode('utf-8') def test_priority_adjust(self): req = Request('http://a.com') @@ -243,7 +243,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req2.method, 'GET') assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" - assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" def test_max_redirect_times(self): self.mw.max_redirect_times = 1 diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 29357ba94..364ce0c89 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -94,7 +94,7 @@ class RetryTest(unittest.TestCase): ] for exc in exceptions: - req = Request('http://www.scrapytest.org/%s' % exc.__name__) + req = Request(f'http://www.scrapytest.org/{exc.__name__}') self._test_retry_exception(req, exc('foo')) stats = self.crawler.stats diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index b9452a0e7..858138f81 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def _get_successful_crawler(self): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) - ROBOTS = u""" + ROBOTS = """ User-Agent: * Disallow: /admin/ Disallow: /static/ @@ -56,7 +56,7 @@ Disallow: /some/randome/page.html self.assertIgnored(Request('http://site.local/admin/main'), middleware), self.assertIgnored(Request('http://site.local/static/'), middleware), self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware), - self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware) + self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware) ], fireOnOneErrback=True) def test_robotstxt_ready_parser(self): @@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): skip = "Rerp parser is not installed" def setUp(self): - super(RobotsTxtMiddlewareWithRerpTest, self).setUp() + super().setUp() self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser') @@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): skip = "Reppy parser is not installed" def setUp(self): - super(RobotsTxtMiddlewareWithReppyTest, self).setUp() + super().setUp() self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser') diff --git a/tests/test_engine.py b/tests/test_engine.py index 1b848ac72..3629aa1aa 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -127,8 +127,8 @@ def start_test_site(debug=False): port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: - print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." - % port.getHost().port) + print(f"Test server running at http://localhost:{port.getHost().port}/ " + "- hit Ctrl-C to finish.") return port @@ -185,7 +185,7 @@ class CrawlerRun: self.deferred.callback(None) def geturl(self, path): - return "http://localhost:%s%s" % (self.portno, path) + return f"http://localhost:{self.portno}{path}" def getpath(self, url): u = urlparse(url) @@ -265,7 +265,7 @@ class EngineTest(unittest.TestCase): "/item1.html", "/item2.html", "/item999.html"] urls_visited = {rp[0].url for rp in self.run.respplug} urls_expected = {self.run.geturl(p) for p in must_be_visited} - assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited) + assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" def _assert_scheduled_requests(self, urls_to_visit=None): self.assertEqual(urls_to_visit, len(self.run.reqplug)) @@ -413,16 +413,19 @@ class StopDownloadEngineTest(EngineTest): yield self.run.run() log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) self._assert_visited_urls() self._assert_scheduled_requests(urls_to_visit=9) self._assert_downloaded_responses() diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b27380309..ebc477e74 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -8,6 +8,7 @@ from io import BytesIO from datetime import datetime import lxml.etree +from itemadapter import ItemAdapter from scrapy.item import Item, Field from scrapy.utils.python import to_unicode @@ -23,10 +24,37 @@ class TestItem(Item): age = Field() +def custom_serializer(value): + return str(int(value) + 2) + + +class CustomFieldItem(Item): + name = Field() + age = Field(serializer=custom_serializer) + + +try: + from dataclasses import make_dataclass, field +except ImportError: + TestDataClass = None + CustomFieldDataclass = None +else: + TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)]) + CustomFieldDataclass = make_dataclass( + "CustomFieldDataclass", + [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))] + ) + + class BaseItemExporterTest(unittest.TestCase): + item_class = TestItem + custom_field_item_class = CustomFieldItem + def setUp(self): - self.i = TestItem(name=u'John\xa3', age=u'22') + if self.item_class is None: + raise unittest.SkipTest("item class is None") + self.i = self.item_class(name='John\xa3', age='22') self.output = BytesIO() self.ie = self._get_exporter() @@ -39,7 +67,7 @@ class BaseItemExporterTest(unittest.TestCase): def _assert_expected_item(self, exported_dict): for k, v in exported_dict.items(): exported_dict[k] = to_unicode(v) - self.assertEqual(self.i, exported_dict) + self.assertEqual(self.i, self.item_class(**exported_dict)) def _get_nonstring_types_item(self): return { @@ -63,37 +91,36 @@ class BaseItemExporterTest(unittest.TestCase): self.assertItemExportWorks(self.i) def test_export_dict_item(self): - self.assertItemExportWorks(dict(self.i)) + self.assertItemExportWorks(ItemAdapter(self.i).asdict()) def test_serialize_field(self): - res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) - self.assertEqual(res, u'John\xa3') + a = ItemAdapter(self.i) + res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name']) + self.assertEqual(res, 'John\xa3') - res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) - self.assertEqual(res, u'22') + res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age']) + self.assertEqual(res, '22') def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=['name']) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')]) + self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')]) ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') _, name = list(ie._get_serialized_fields(self.i))[0] assert isinstance(name, str) - self.assertEqual(name, u'John\xa3') + self.assertEqual(name, 'John\xa3') def test_field_custom_serializer(self): - def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): - name = Field() - age = Field(serializer=custom_serializer) - - i = CustomFieldItem(name=u'John\xa3', age=u'22') - + i = self.custom_field_item_class(name='John\xa3', age='22') + a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3') + self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24') + + +class BaseItemExporterDataclassTest(BaseItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass class PythonItemExporterTest(BaseItemExporterTest): @@ -105,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest): PythonItemExporter(invalid_option='something') def test_nested_item(self): - i1 = TestItem(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) self.assertEqual( exported, - {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'} + {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'} ) self.assertEqual(type(exported['age']), dict) self.assertEqual(type(exported['age']['age']), dict) def test_export_list(self): - i1 = TestItem(name=u'Joseph', age='22') - i2 = TestItem(name=u'Maria', age=[i1]) - i3 = TestItem(name=u'Jesus', age=[i2]) + i1 = self.item_class(name='Joseph', age='22') + i2 = self.item_class(name='Maria', age=[i1]) + i3 = self.item_class(name='Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( exported, - {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'} ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_item_dict_list(self): - i1 = TestItem(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=[i1]) - i3 = TestItem(name=u'Jesus', age=[i2]) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=[i1]) + i3 = self.item_class(name='Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( exported, - {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'} ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_binary(self): exporter = PythonItemExporter(binary=True) - value = TestItem(name=u'John\xa3', age=u'22') + value = self.item_class(name='John\xa3', age='22') expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) @@ -157,6 +184,11 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, item) +class PythonItemExporterDataclassTest(PythonItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class PprintItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -166,6 +198,11 @@ class PprintItemExporterTest(BaseItemExporterTest): self._assert_expected_item(eval(self.output.getvalue())) +class PprintItemExporterDataclassTest(PprintItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class PickleItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -175,8 +212,8 @@ class PickleItemExporterTest(BaseItemExporterTest): self._assert_expected_item(pickle.loads(self.output.getvalue())) def test_export_multiple_items(self): - i1 = TestItem(name='hello', age='world') - i2 = TestItem(name='bye', age='world') + i1 = self.item_class(name='hello', age='world') + i2 = self.item_class(name='bye', age='world') f = BytesIO() ie = PickleItemExporter(f) ie.start_exporting() @@ -184,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(i2) ie.finish_exporting() f.seek(0) - self.assertEqual(pickle.load(f), i1) - self.assertEqual(pickle.load(f), i2) + self.assertEqual(self.item_class(**pickle.load(f)), i1) + self.assertEqual(self.item_class(**pickle.load(f)), i2) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -197,6 +234,11 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.loads(fp.getvalue()), item) +class PickleItemExporterDataclassTest(PickleItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class MarshalItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -219,6 +261,11 @@ class MarshalItemExporterTest(BaseItemExporterTest): self.assertEqual(marshal.load(fp), item) +class MarshalItemExporterDataclassTest(MarshalItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) @@ -232,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest): return self.assertEqual(split_csv(first), split_csv(second), msg=msg) def _check_output(self): - self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') + self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() @@ -245,18 +292,18 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_header_export_all(self): self.assertExportResult( item=self.i, - fields_to_export=self.i.fields.keys(), + fields_to_export=ItemAdapter(self.i).field_names(), expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_all_dict(self): self.assertExportResult( - item=dict(self.i), + item=ItemAdapter(self.i).asdict(), expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_single_field(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: self.assertExportResult( item=item, fields_to_export=['age'], @@ -264,7 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ) def test_header_export_two_items(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: output = BytesIO() ie = CsvItemExporter(output) ie.start_exporting() @@ -275,7 +322,7 @@ class CsvItemExporterTest(BaseItemExporterTest): b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') def test_header_no_header_line(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: self.assertExportResult( item=item, include_headers_line=False, @@ -308,6 +355,28 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='22,False,3.14,2015-01-01 01:01:01\r\n' ) + def test_errors_default(self): + with self.assertRaises(UnicodeEncodeError): + self.assertExportResult( + item=dict(text=u'W\u0275\u200Brd'), + expected=None, + encoding='windows-1251', + ) + + def test_errors_xmlcharrefreplace(self): + self.assertExportResult( + item=dict(text=u'W\u0275\u200Brd'), + include_headers_line=False, + expected='Wɵ​rd\r\n', + encoding='windows-1251', + errors='xmlcharrefreplace', + ) + + +class CsvItemExporterDataclassTest(CsvItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + class XmlItemExporterTest(BaseItemExporterTest): @@ -318,8 +387,7 @@ class XmlItemExporterTest(BaseItemExporterTest): def xmltuple(elem): children = list(elem.iterchildren()) if children: - return [(child.tag, sorted(xmltuple(child))) - for child in children] + return [(child.tag, sorted(xmltuple(child))) for child in children] else: return [(elem.tag, [(elem.text, ())])] @@ -345,17 +413,21 @@ class XmlItemExporterTest(BaseItemExporterTest): def test_multivalued_fields(self): self.assertExportResult( - TestItem(name=[u'John\xa3', u'Doe']), - ( - b'\n' - b'John\xc2\xa3Doe' - ) + self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]), + b"""\n + + + John\xc2\xa3Doe + 123 + + + """ ) def test_nested_item(self): - i1 = TestItem(name=u'foo\xa3hoo', age='22') - i2 = dict(name=u'bar', age=i1) - i3 = TestItem(name=u'buz', age=i2) + i1 = dict(name='foo\xa3hoo', age='22') + i2 = dict(name='bar', age=i1) + i3 = self.item_class(name='buz', age=i2) self.assertExportResult( i3, @@ -376,9 +448,9 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = TestItem(name=u'foo') - i2 = dict(name=u'bar', v2={"egg": ["spam"]}) - i3 = TestItem(name=u'buz', age=[i1, i2]) + i1 = dict(name='foo') + i2 = dict(name='bar', v2={"egg": ["spam"]}) + i3 = self.item_class(name='buz', age=[i1, i2]) self.assertExportResult( i3, @@ -412,21 +484,27 @@ class XmlItemExporterTest(BaseItemExporterTest): ) +class XmlItemExporterDataclassTest(XmlItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class JsonLinesItemExporterTest(BaseItemExporterTest): - _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} + _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} def _get_exporter(self, **kwargs): return JsonLinesItemExporter(self.output, **kwargs) def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, dict(self.i)) + self.assertEqual(exported, ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = TestItem(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() @@ -449,6 +527,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, item) +class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class JsonItemExporterTest(JsonLinesItemExporterTest): _expected_nested = [JsonLinesItemExporterTest._expected_nested] @@ -458,7 +542,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, [dict(self.i)]) + self.assertEqual(exported, [ItemAdapter(self.i).asdict()]) def assertTwoItemsExported(self, item): self.ie.start_exporting() @@ -466,34 +550,34 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, [dict(item), dict(item)]) + self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()]) def test_two_items(self): self.assertTwoItemsExported(self.i) def test_two_dict_items(self): - self.assertTwoItemsExported(dict(self.i)) + self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = TestItem(name=u'Joseph\xa3', age='22') - i2 = TestItem(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph\xa3', age='22') + i2 = self.item_class(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}} + expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} self.assertEqual(exported, [expected]) def test_nested_dict_item(self): - i1 = dict(name=u'Joseph\xa3', age='22') - i2 = TestItem(name=u'Maria', age=i1) - i3 = dict(name=u'Jesus', age=i2) + i1 = dict(name='Joseph\xa3', age='22') + i2 = self.item_class(name='Maria', age=i1) + i3 = dict(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}} + expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected]) def test_nonstring_types_item(self): @@ -506,7 +590,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertEqual(exported, [item]) -class CustomItemExporterTest(unittest.TestCase): +class JsonItemExporterDataclassTest(JsonItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + +class CustomExporterItemTest(unittest.TestCase): + + item_class = TestItem + + def setUp(self): + if self.item_class is None: + raise unittest.SkipTest("item class is None") def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): @@ -514,18 +610,24 @@ class CustomItemExporterTest(unittest.TestCase): if name == 'age': return str(int(value) + 1) else: - return super(CustomItemExporter, self).serialize_field(field, name, value) + return super().serialize_field(field, name, value) - i = TestItem(name=u'John', age='22') + i = self.item_class(name='John', age='22') + a = ItemAdapter(i) ie = CustomItemExporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John') + self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23') - i2 = {'name': u'John', 'age': '22'} + i2 = {'name': 'John', 'age': '22'} self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John') self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23') +class CustomExporterDataclassTest(CustomExporterItemTest): + + item_class = TestDataClass + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 37384081a..840e0f87b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -6,6 +6,8 @@ import shutil import string import tempfile import warnings +from abc import ABC, abstractmethod +from collections import defaultdict from io import BytesIO from logging import getLogger from pathlib import Path @@ -24,9 +26,11 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, + FeedExporter, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, @@ -43,7 +47,7 @@ from scrapy.utils.test import ( mock_google_cloud_storage, ) -from tests.mockserver import MockServer +from tests.mockserver import MockFTPServer, MockServer class FileFeedStorageTest(unittest.TestCase): @@ -72,8 +76,28 @@ class FileFeedStorageTest(unittest.TestCase): st = FileFeedStorage(path) verifyObject(IFeedStorage, st) + def _store(self, feed_options=None): + path = os.path.abspath(self.mktemp()) + storage = FileFeedStorage(path, feed_options=feed_options) + spider = scrapy.Spider("default") + file = storage.open(spider) + file.write(b"content") + storage.store(file) + return path + + def test_append(self): + path = self._store() + return self._assert_stores(FileFeedStorage(path), path, b"contentcontent") + + def test_overwrite(self): + path = self._store({"overwrite": True}) + return self._assert_stores( + FileFeedStorage(path, feed_options={"overwrite": True}), + path + ) + @defer.inlineCallbacks - def _assert_stores(self, storage, path): + def _assert_stores(self, storage, path, expected_content=b"content"): spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") @@ -81,7 +105,7 @@ class FileFeedStorageTest(unittest.TestCase): self.assertTrue(os.path.exists(path)) try: with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"content") + self.assertEqual(fp.read(), expected_content) finally: os.unlink(path) @@ -91,59 +115,85 @@ class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = 'test_spider' + crawler = get_crawler(settings_dict=settings) spider = TestSpider.from_crawler(crawler) return spider - def test_store(self): - uri = os.environ.get('FEEDTEST_FTP_URI') - path = os.environ.get('FEEDTEST_FTP_PATH') - if not (uri and path): - raise unittest.SkipTest("No FTP server available for testing") - st = FTPFeedStorage(uri) - verifyObject(IFeedStorage, st) - return self._assert_stores(st, path) + def _store(self, uri, content, feed_options=None, settings=None): + crawler = get_crawler(settings_dict=settings or {}) + storage = FTPFeedStorage.from_crawler( + crawler, + uri, + feed_options=feed_options, + ) + verifyObject(IFeedStorage, storage) + spider = self.get_test_spider() + file = storage.open(spider) + file.write(content) + return storage.store(file) - def test_store_active_mode(self): - uri = os.environ.get('FEEDTEST_FTP_URI') - path = os.environ.get('FEEDTEST_FTP_PATH') - if not (uri and path): - raise unittest.SkipTest("No FTP server available for testing") - use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True} - crawler = get_crawler(settings_dict=use_active_mode) - st = FTPFeedStorage.from_crawler(crawler, uri) - verifyObject(IFeedStorage, st) - return self._assert_stores(st, path) + def _assert_stored(self, path, content): + self.assertTrue(path.exists()) + try: + with path.open('rb') as fp: + self.assertEqual(fp.read(), content) + finally: + os.unlink(str(path)) + + @defer.inlineCallbacks + def test_append(self): + with MockFTPServer() as ftp_server: + filename = 'file' + url = ftp_server.url(filename) + feed_options = {'overwrite': False} + yield self._store(url, b"foo", feed_options=feed_options) + yield self._store(url, b"bar", feed_options=feed_options) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @defer.inlineCallbacks + def test_overwrite(self): + with MockFTPServer() as ftp_server: + filename = 'file' + url = ftp_server.url(filename) + yield self._store(url, b"foo") + yield self._store(url, b"bar") + self._assert_stored(ftp_server.path / filename, b"bar") + + @defer.inlineCallbacks + def test_append_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {'FEED_STORAGE_FTP_ACTIVE': True} + filename = 'file' + url = ftp_server.url(filename) + feed_options = {'overwrite': False} + yield self._store(url, b"foo", feed_options=feed_options, settings=settings) + yield self._store(url, b"bar", feed_options=feed_options, settings=settings) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @defer.inlineCallbacks + def test_overwrite_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {'FEED_STORAGE_FTP_ACTIVE': True} + filename = 'file' + url = ftp_server.url(filename) + yield self._store(url, b"foo", settings=settings) + yield self._store(url, b"bar", settings=settings) + self._assert_stored(ftp_server.path / filename, b"bar") def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe='') - st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted) + st = FTPFeedStorage(f'ftp://foo:{pw_quoted}@example.com/some_path', {}) self.assertEqual(st.password, string.punctuation) - @defer.inlineCallbacks - def _assert_stores(self, storage, path): - spider = self.get_test_spider() - file = storage.open(spider) - file.write(b"content") - yield storage.store(file) - self.assertTrue(os.path.exists(path)) - try: - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"content") - # again, to check s3 objects are overwritten - yield storage.store(BytesIO(b"new content")) - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"new content") - finally: - os.unlink(path) - class BlockingFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = 'test_spider' + crawler = get_crawler(settings_dict=settings) spider = TestSpider.from_crawler(crawler) return spider @@ -176,21 +226,19 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): - @mock.patch('scrapy.utils.project.get_project_settings', - new=mock.MagicMock(return_value={'AWS_ACCESS_KEY_ID': 'conf_key', - 'AWS_SECRET_ACCESS_KEY': 'conf_secret'}), - create=True) def test_parse_credentials(self): try: - import boto # noqa: F401 + import botocore # noqa: F401 except ImportError: - raise unittest.SkipTest("S3FeedStorage requires boto") + raise unittest.SkipTest("S3FeedStorage requires botocore") aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} crawler = get_crawler(settings_dict=aws_credentials) # Instantiate with crawler - storage = S3FeedStorage.from_crawler(crawler, - 's3://mybucket/export.csv') + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv', + ) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') # Instantiate directly @@ -205,12 +253,6 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials['AWS_SECRET_ACCESS_KEY']) self.assertEqual(storage.access_key, 'uri_key') self.assertEqual(storage.secret_key, 'uri_secret') - # Backward compatibility for initialising without settings - with warnings.catch_warnings(record=True) as w: - storage = S3FeedStorage('s3://mybucket/export.csv') - self.assertEqual(storage.access_key, 'conf_key') - self.assertEqual(storage.secret_key, 'conf_secret') - self.assertTrue('without AWS keys' in str(w[-1].message)) @defer.inlineCallbacks def test_store(self): @@ -259,7 +301,7 @@ class S3FeedStorageTest(unittest.TestCase): crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv' + 's3://mybucket/export.csv', ) self.assertEqual(storage.access_key, 'access_key') self.assertEqual(storage.secret_key, 'secret_key') @@ -274,7 +316,7 @@ class S3FeedStorageTest(unittest.TestCase): crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv' + 's3://mybucket/export.csv', ) self.assertEqual(storage.access_key, 'access_key') self.assertEqual(storage.secret_key, 'secret_key') @@ -375,6 +417,27 @@ class S3FeedStorageTest(unittest.TestCase): key.set_contents_from_file.call_args ) + def test_overwrite_default(self): + with LogCapture() as log: + S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertNotIn('S3 does not support appending to files', str(log)) + + def test_overwrite_false(self): + with LogCapture() as log: + S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl', + feed_options={'overwrite': False}, + ) + self.assertIn('S3 does not support appending to files', str(log)) + class GCSFeedStorageTest(unittest.TestCase): @@ -444,12 +507,22 @@ class StdoutFeedStorageTest(unittest.TestCase): yield storage.store(file) self.assertEqual(out.getvalue(), b"content") + def test_overwrite_default(self): + with LogCapture() as log: + StdoutFeedStorage('stdout:') + self.assertNotIn('Standard output (stdout) storage does not support overwriting', str(log)) + + def test_overwrite_true(self): + with LogCapture() as log: + StdoutFeedStorage('stdout:', feed_options={'overwrite': True}) + self.assertIn('Standard output (stdout) storage does not support overwriting', str(log)) + class FromCrawlerMixin: init_with_crawler = False @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler, *args, feed_options=None, **kwargs): cls.init_with_crawler = True return cls(*args, **kwargs) @@ -459,7 +532,11 @@ class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin): class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): - pass + + @classmethod + def from_crawler(cls, crawler, *args, feed_options=None, **kwargs): + cls.init_with_crawler = True + return cls(*args, feed_options=feed_options, **kwargs) class DummyBlockingFeedStorage(BlockingFeedStorage): @@ -502,68 +579,31 @@ class LogOnStoreFileStorage: file.close() -class FeedExportTest(unittest.TestCase): +class FeedExportTestBase(ABC, unittest.TestCase): + __test__ = False class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() baz = scrapy.Field() + def _random_temp_filename(self, inter_dir=''): + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = ''.join(chars) + return os.path.join(self.temp_dir, inter_dir, filename) + def setUp(self): self.temp_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.temp_dir, ignore_errors=True) - def _random_temp_filename(self): - chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = ''.join(chars) - return os.path.join(self.temp_dir, filename) - - @defer.inlineCallbacks - def run_and_export(self, spider_cls, settings): - """ Run spider with specified settings; return exported data. """ - - def path_to_url(path): - return urljoin('file:', pathname2url(str(path))) - - def printf_escape(string): - return string.replace('%', '%%') - - FEEDS = settings.get('FEEDS') or {} - settings['FEEDS'] = { - printf_escape(path_to_url(file_path)): feed - for file_path, feed in FEEDS.items() - } - - content = {} - try: - with MockServer() as s: - runner = CrawlerRunner(Settings(settings)) - spider_cls.start_urls = [s.url('/')] - yield runner.crawl(spider_cls) - - for file_path, feed in FEEDS.items(): - if not os.path.exists(str(file_path)): - continue - - with open(str(file_path), 'rb') as f: - content[feed['format']] = f.read() - - finally: - for file_path in FEEDS.keys(): - if not os.path.exists(str(file_path)): - continue - - os.remove(str(file_path)) - - return content - @defer.inlineCallbacks def exported_data(self, items, settings): """ Return exported data which a spider yielding ``items`` would return. """ + class TestSpider(scrapy.Spider): name = 'testspider' @@ -579,6 +619,7 @@ class FeedExportTest(unittest.TestCase): """ Return exported data which a spider yielding no ``items`` would return. """ + class TestSpider(scrapy.Spider): name = 'testspider' @@ -588,6 +629,74 @@ class FeedExportTest(unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) return data + @defer.inlineCallbacks + def assertExported(self, items, header, rows, settings=None, ordered=True): + yield self.assertExportedCsv(items, header, rows, settings, ordered) + yield self.assertExportedJsonLines(items, rows, settings) + yield self.assertExportedXml(items, rows, settings) + yield self.assertExportedPickle(items, rows, settings) + yield self.assertExportedMarshal(items, rows, settings) + yield self.assertExportedMultiple(items, rows, settings) + + @abstractmethod + def run_and_export(self, spider_cls, settings): + pass + + def _load_until_eof(self, data, load_func): + result = [] + with tempfile.TemporaryFile() as temp: + temp.write(data) + temp.seek(0) + while True: + try: + result.append(load_func(temp)) + except EOFError: + break + return result + + +class FeedExportTest(FeedExportTestBase): + __test__ = True + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings): + """ Run spider with specified settings; return exported data. """ + + def path_to_url(path): + return urljoin('file:', pathname2url(str(path))) + + def printf_escape(string): + return string.replace('%', '%%') + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() + } + + content = {} + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + spider_cls.start_urls = [s.url('/')] + yield runner.crawl(spider_cls) + + for file_path, feed_options in FEEDS.items(): + if not os.path.exists(str(file_path)): + continue + + with open(str(file_path), 'rb') as f: + content[feed_options['format']] = f.read() + + finally: + for file_path in FEEDS.keys(): + if not os.path.exists(str(file_path)): + continue + + os.remove(str(file_path)) + + return content + @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -653,18 +762,6 @@ class FeedExportTest(unittest.TestCase): json_rows = json.loads(to_unicode(data['json'])) self.assertEqual(rows, json_rows) - def _load_until_eof(self, data, load_func): - result = [] - with tempfile.TemporaryFile() as temp: - temp.write(data) - temp.seek(0) - while True: - try: - result.append(load_func(temp)) - except EOFError: - break - return result - @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} @@ -693,15 +790,6 @@ class FeedExportTest(unittest.TestCase): result = self._load_until_eof(data['marshal'], load_func=marshal.load) self.assertEqual(expected, result) - @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None, ordered=True): - yield self.assertExportedCsv(items, header, rows, settings, ordered) - yield self.assertExportedJsonLines(items, rows, settings) - yield self.assertExportedXml(items, rows, settings) - yield self.assertExportedPickle(items, rows, settings) - yield self.assertExportedMarshal(items, rows, settings) - yield self.assertExportedMultiple(items, rows, settings) - @defer.inlineCallbacks def test_export_items(self): # feed exporters use field names from Item @@ -725,7 +813,7 @@ class FeedExportTest(unittest.TestCase): }, } data = yield self.exported_no_data(settings) - self.assertEqual(data[fmt], b'') + self.assertEqual(b'', data[fmt]) @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -745,7 +833,7 @@ class FeedExportTest(unittest.TestCase): 'FEED_EXPORT_INDENT': None, } data = yield self.exported_no_data(settings) - self.assertEqual(data[fmt], expctd) + self.assertEqual(expctd, data[fmt]) @defer.inlineCallbacks def test_export_no_items_multiple_feeds(self): @@ -756,7 +844,7 @@ class FeedExportTest(unittest.TestCase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'}, + 'FEED_STORAGES': {'file': LogOnStoreFileStorage}, 'FEED_STORE_EMPTY': False } @@ -857,7 +945,7 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_encoding(self): - items = [dict({'foo': u'Test\xd6'})] + items = [dict({'foo': 'Test\xd6'})] formats = { 'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'), @@ -902,7 +990,7 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'})] + items = [dict({'foo': 'FOO', 'bar': 'BAR'})] formats = { 'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'), @@ -1100,8 +1188,8 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_init_exporters_storages_with_crawler(self): settings = { - 'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.FromCrawlerCsvItemExporter'}, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.FromCrawlerFileFeedStorage'}, + 'FEED_EXPORTERS': {'csv': FromCrawlerCsvItemExporter}, + 'FEED_STORAGES': {'file': FromCrawlerFileFeedStorage}, 'FEEDS': { self._random_temp_filename(): {'format': 'csv'}, }, @@ -1130,7 +1218,7 @@ class FeedExportTest(unittest.TestCase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'}, + 'FEED_STORAGES': {'file': DummyBlockingFeedStorage}, } items = [ {'foo': 'bar1', 'baz': ''}, @@ -1141,7 +1229,7 @@ class FeedExportTest(unittest.TestCase): print(log) for fmt in ['json', 'xml', 'csv']: - self.assertIn('Stored %s feed (2 items)' % fmt, str(log)) + self.assertIn(f'Stored {fmt} feed (2 items)', str(log)) @defer.inlineCallbacks def test_multiple_feeds_failing_logs_blocking_feed_storage(self): @@ -1151,7 +1239,7 @@ class FeedExportTest(unittest.TestCase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'}, + 'FEED_STORAGES': {'file': FailingBlockingFeedStorage}, } items = [ {'foo': 'bar1', 'baz': ''}, @@ -1162,4 +1250,634 @@ class FeedExportTest(unittest.TestCase): print(log) for fmt in ['json', 'xml', 'csv']: - self.assertIn('Error storing %s feed (2 items)' % fmt, str(log)) + self.assertIn(f'Error storing {fmt} feed (2 items)', str(log)) + + +class BatchDeliveriesTest(FeedExportTestBase): + __test__ = True + _file_mark = '_%(batch_time)s_#%(batch_id)02d_' + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings): + """ Run spider with specified settings; return exported data. """ + + def build_url(path): + if path[0] != '/': + path = '/' + path + return urljoin('file:', path) + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + build_url(file_path): feed + for file_path, feed in FEEDS.items() + } + content = defaultdict(list) + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + spider_cls.start_urls = [s.url('/')] + yield runner.crawl(spider_cls) + + for path, feed in FEEDS.items(): + dir_name = os.path.dirname(path) + for file in sorted(os.listdir(dir_name)): + with open(os.path.join(dir_name, file), 'rb') as f: + data = f.read() + content[feed['format']].append(data) + finally: + self.tearDown() + defer.returnValue(content) + + @defer.inlineCallbacks + def assertExportedJsonLines(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, + }, + }) + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + for batch in data['jl']: + got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, + }, + }) + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + data = yield self.exported_data(items, settings) + for batch in data['csv']: + got_batch = csv.DictReader(to_unicode(batch).splitlines()) + self.assertEqual(list(header), got_batch.fieldnames) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, list(got_batch)) + + @defer.inlineCallbacks + def assertExportedXml(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, + }, + }) + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + for batch in data['xml']: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedMultiple(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, + os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, + }, + }) + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + # XML + xml_rows = rows.copy() + for batch in data['xml']: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')] + expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + # JSON + json_rows = rows.copy() + for batch in data['json']: + got_batch = json.loads(batch.decode('utf-8')) + expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedPickle(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, + }, + }) + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + import pickle + for batch in data['pickle']: + got_batch = self._load_until_eof(batch, load_func=pickle.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedMarshal(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, + }, + }) + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + import marshal + for batch in data['marshal']: + got_batch = self._load_until_eof(batch, load_func=marshal.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def test_export_items(self): + """ Test partial deliveries in all supported formats """ + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + rows = [ + {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, + {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}, + {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} + ] + settings = { + 'FEED_EXPORT_BATCH_ITEM_COUNT': 2 + } + header = self.MyItem.fields.keys() + yield self.assertExported(items, header, rows, settings=Settings(settings)) + + def test_wrong_path(self): + """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """ + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': 'xml'}, + }, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 + } + crawler = get_crawler(settings_dict=settings) + self.assertRaises(NotConfigured, FeedExporter, crawler) + + @defer.inlineCallbacks + def test_export_no_items_not_store_empty(self): + for fmt in ('json', 'jsonlines', 'xml', 'csv'): + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + }, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 + } + data = yield self.exported_no_data(settings) + data = dict(data) + self.assertEqual(b'', data[fmt][0]) + + @defer.inlineCallbacks + def test_export_no_items_store_empty(self): + formats = ( + ('json', b'[]'), + ('jsonlines', b''), + ('xml', b'\n'), + ('csv', b''), + ) + + for fmt, expctd in formats: + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + }, + 'FEED_STORE_EMPTY': True, + 'FEED_EXPORT_INDENT': None, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, + } + data = yield self.exported_no_data(settings) + data = dict(data) + self.assertEqual(expctd, data[fmt][0]) + + @defer.inlineCallbacks + def test_export_multiple_configs(self): + items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})] + + formats = { + 'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'), + '[\n{"bar": "BAR1"}\n]'.encode('utf-8')], + 'xml': [ + ( + '\n' + '\n \n FOO\n \n' + ).encode('latin-1'), + ( + '\n' + '\n \n FOO1\n \n' + ).encode('latin-1') + ], + 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'), + 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')], + } + + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + 'format': 'json', + 'indent': 0, + 'fields': ['bar'], + 'encoding': 'utf-8', + }, + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): { + 'format': 'xml', + 'indent': 2, + 'fields': ['foo'], + 'encoding': 'latin-1', + }, + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + 'format': 'csv', + 'indent': None, + 'fields': ['foo', 'bar'], + 'encoding': 'utf-8', + }, + }, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, + } + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt]): + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def test_batch_item_count_feeds_setting(self): + items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})] + formats = { + 'json': ['[{"foo": "FOO"}]'.encode('utf-8'), + '[{"foo": "FOO1"}]'.encode('utf-8')], + } + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + 'format': 'json', + 'indent': None, + 'encoding': 'utf-8', + 'batch_item_count': 1, + }, + }, + } + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt]): + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def test_batch_path_differ(self): + """ + Test that the name of all batch files differ from each other. + So %(batch_time)s replaced with the current date. + """ + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), '%(batch_time)s'): { + 'format': 'json', + }, + }, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, + } + data = yield self.exported_data(items, settings) + self.assertEqual(len(items) + 1, len(data['json'])) + + @defer.inlineCallbacks + def test_s3_export(self): + """ + Test export of items into s3 bucket. + S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini + to perform this test: + [testenv] + setenv = + AWS_SECRET_ACCESS_KEY = ABCD + AWS_ACCESS_KEY_ID = EFGH + S3_TEST_BUCKET_NAME = IJKL + """ + try: + import boto3 + except ImportError: + raise unittest.SkipTest("S3FeedStorage requires boto3") + + assert_aws_environ() + s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME') + access_key = os.environ.get('AWS_ACCESS_KEY_ID') + secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') + if not s3_test_bucket_name: + raise unittest.SkipTest("No S3 BUCKET available for testing") + + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = ''.join(chars) + prefix = f'tmp/{filename}' + s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json' + storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) + settings = Settings({ + 'FEEDS': { + s3_test_file_uri: { + 'format': 'json', + }, + }, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, + }) + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + verifyObject(IFeedStorage, storage) + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + for item in items: + yield item + + s3 = boto3.resource('s3') + my_bucket = s3.Bucket(s3_test_bucket_name) + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + TestSpider.start_urls = [s.url('/')] + yield runner.crawl(TestSpider) + + for file_uri in my_bucket.objects.filter(Prefix=prefix): + content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key) + if not content and not items: + break + content = json.loads(content.decode('utf-8')) + expected_batch, items = items[:batch_size], items[batch_size:] + self.assertEqual(expected_batch, content) + + +class FeedExportInitTest(unittest.TestCase): + + def test_unsupported_storage(self): + settings = { + 'FEEDS': { + 'unsupported://uri': {}, + }, + } + crawler = get_crawler(settings_dict=settings) + with self.assertRaises(NotConfigured): + FeedExporter.from_crawler(crawler) + + def test_unsupported_format(self): + settings = { + 'FEEDS': { + 'file://path': { + 'format': 'unsupported_format', + }, + }, + } + crawler = get_crawler(settings_dict=settings) + with self.assertRaises(NotConfigured): + FeedExporter.from_crawler(crawler) + + +class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): + + def __init__(self, uri): + super().__init__(uri) + + +class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': StdoutFeedStorageWithoutFeedOptions + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "StdoutFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class FileFeedStorageWithoutFeedOptions(FileFeedStorage): + + def __init__(self, uri): + super().__init__(uri) + + +class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': FileFeedStorageWithoutFeedOptions + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FileFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class S3FeedStorageWithoutFeedOptions(S3FeedStorage): + + def __init__(self, uri, access_key, secret_key, acl): + super().__init__(uri, access_key, secret_key, acl) + + +class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): + + @classmethod + def from_crawler(cls, crawler, uri): + return super().from_crawler(crawler, uri) + + +class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': S3FeedStorageWithoutFeedOptions + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "S3FeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + def test_from_crawler(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " + "does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): + + def __init__(self, uri, use_active_mode=False): + super().__init__(uri) + + +class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage): + + @classmethod + def from_crawler(cls, crawler, uri): + return super().from_crawler(crawler, uri) + + +class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': FTPFeedStorageWithoutFeedOptions + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FTPFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + def test_from_crawler(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " + "does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index cf3fc8496..64ff7a73d 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase): assert h.getlist('X-Forwarded-For') is not hlist def test_encode_utf8(self): - h = Headers({u'key': u'\xa3'}, encoding='utf-8') + h = Headers({'key': '\xa3'}, encoding='utf-8') key, val = dict(h).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] self.assertEqual(val[0], b'\xc2\xa3') def test_encode_latin1(self): - h = Headers({u'key': u'\xa3'}, encoding='latin1') + h = Headers({'key': '\xa3'}, encoding='latin1') key, val = dict(h).popitem() self.assertEqual(val[0], b'\xa3') def test_encode_multiple(self): - h = Headers({u'key': [u'\xa3']}, encoding='utf-8') + h = Headers({'key': ['\xa3']}, encoding='utf-8') key, val = dict(h).popitem() self.assertEqual(val[0], b'\xc2\xa3') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 63014b22d..0a303dbe2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase): self.assertFalse(p.headers is r.headers) # headers must not be unicode - h = Headers({'key1': u'val1', u'key2': 'val2'}) - h[u'newkey'] = u'newval' + h = Headers({'key1': 'val1', 'key2': 'val2'}) + h['newkey'] = 'newval' for k, v in h.items(): self.assertIsInstance(k, bytes) for s in v: @@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase): self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") def test_url_encoding(self): - r = self.request_class(url=u"http://www.scrapy.org/price/£") + r = self.request_class(url="http://www.scrapy.org/price/£") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") def test_url_encoding_other(self): # encoding affects only query part of URI, not path # path part should always be UTF-8 encoded before percent-escaping - r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8") + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") - r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1") + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") def test_url_encoding_query(self): - r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ") + r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") # should be same as above - r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") + r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") def test_url_encoding_query_latin1(self): # encoding is used for encoding query-string before percent-escaping; # path is still UTF-8 encoded before percent-escaping - r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1") + r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1") self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") def test_url_encoding_nonutf8_untouched(self): @@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase): # characters. Otherwise, in the future the IRI will be mapped to # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different # URI from "http://www.example.org/r%E9sum%E9.html". - r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3") + r1 = self.request_class(url="http://www.scrapy.org/price/%a3") self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3") - r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3") + r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html") + r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html") def test_body(self): @@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase): assert isinstance(r2.body, bytes) self.assertEqual(r2.encoding, 'utf-8') # default encoding - r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8') + r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8') assert isinstance(r3.body, bytes) self.assertEqual(r3.body, b"Price: \xc2\xa3100") - r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1') + r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1') assert isinstance(r4.body, bytes) self.assertEqual(r4.body, b"Price: \xa3100") @@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase): r = self.request_class(url="http://www.example.com/ajax.html#!key=value") self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") # unicode url - r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value") + r = self.request_class(url="http://www.example.com/ajax.html#!key=value") self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") def test_copy(self): @@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase): assert r4.dont_filter is False def test_method_always_str(self): - r = self.request_class("http://www.example.com", method=u"POST") + r = self.request_class("http://www.example.com", method="POST") assert isinstance(r.method, str) def test_immutable_attributes(self): @@ -381,7 +381,7 @@ class FormRequestTest(RequestTest): def test_default_encoding_textual_data(self): # using default encoding (utf-8) - data = {u'µ one': u'two', u'price': u'£ 100'} + data = {'µ one': 'two', 'price': '£ 100'} r2 = self.request_class("http://www.example.com", formdata=data) self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'utf-8') @@ -390,7 +390,7 @@ class FormRequestTest(RequestTest): def test_default_encoding_mixed_data(self): # using default encoding (utf-8) - data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'} + data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'} r2 = self.request_class("http://www.example.com", formdata=data) self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'utf-8') @@ -406,14 +406,14 @@ class FormRequestTest(RequestTest): self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') def test_custom_encoding_textual_data(self): - data = {'price': u'£ 100'} + data = {'price': '£ 100'} r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') self.assertEqual(r3.encoding, 'latin1') self.assertEqual(r3.body, b'price=%A3+100') def test_multi_key_values(self): # using multiples values for a single key - data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']} + data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']} r3 = self.request_class("http://www.example.com", formdata=data) self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100') @@ -450,10 +450,10 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_post_nonascii_bytes_latin1(self): response = _buildresponse( @@ -471,14 +471,14 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_post_nonascii_unicode(self): response = _buildresponse( - u""" + """ @@ -490,10 +490,10 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_duplicate_form_key(self): response = _buildresponse( @@ -685,7 +685,7 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response( - response, clickdata={u'name': u'clickable', u'value': u'clicked2'} + response, clickdata={'name': 'clickable', 'value': 'clicked2'} ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) @@ -694,21 +694,21 @@ class FormRequestTest(RequestTest): def test_from_response_unicode_clickdata(self): response = _buildresponse( - u"""
    + """
    """) req = self.request_class.from_response( - response, clickdata={u'name': u'price in \u00a3'} + response, clickdata={'name': 'price in \u00a3'} ) fs = _qs(req, to_unicode=True) - self.assertTrue(fs[u'price in \u00a3']) + self.assertTrue(fs['price in \u00a3']) def test_from_response_unicode_clickdata_latin1(self): response = _buildresponse( - u"""
    + """ @@ -716,10 +716,10 @@ class FormRequestTest(RequestTest):
    """, encoding='latin1') req = self.request_class.from_response( - response, clickdata={u'name': u'price in \u00a5'} + response, clickdata={'name': 'price in \u00a5'} ) fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertTrue(fs[u'price in \u00a5']) + self.assertTrue(fs['price in \u00a5']) def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( @@ -733,7 +733,7 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response( - response, formname='form2', clickdata={u'name': u'clickable'} + response, formname='form2', clickdata={'name': 'clickable'} ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) @@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest): def test_from_response_unicode_xpath(self): response = _buildresponse(b'
    ') - r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']") + r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']") fs = _qs(r) self.assertEqual(fs, {}) - xpath = u"//form[@name='\u03b1']" + xpath = "//form[@name='\u03b1']" self.assertRaisesRegex(ValueError, re.escape(xpath), self.request_class.from_response, response, formxpath=xpath) @@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=('value',)) self._test_request(params=('username', 'password'), methodname='login') self._test_request(params=('response', ), methodresponse='login') - self._test_request(params=(u'pas£',), encoding='utf-8') + self._test_request(params=('pas£',), encoding='utf-8') self._test_request(params=(None,), allow_none=1) self.assertRaises(TypeError, self._test_request) self.assertRaises(TypeError, self._test_request, params=(None,)) def test_latin1(self): - self._test_request(params=(u'pas£',), encoding='latin1') + self._test_request(params=('pas£',), encoding='latin1') class JsonRequestTest(RequestTest): @@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest): def setUp(self): warnings.simplefilter("always") - super(JsonRequestTest, self).setUp() + super().setUp() def test_data(self): r1 = self.request_class(url="http://www.example.com/") @@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest): def tearDown(self): warnings.resetwarnings() - super(JsonRequestTest, self).tearDown() + super().tearDown() if __name__ == "__main__": diff --git a/tests/test_http_response.py b/tests/test_http_response.py index e0ca3c0e6..f831ef5dc 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest): response_class = TextResponse def test_replace(self): - super(TextResponseTest, self).test_replace() + super().test_replace() r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852") r2 = r1.replace(url="http://www.example.com/other") r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") @@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest): def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) - resp = self.response_class(u"http://www.example.com/") + resp = self.response_class("http://www.example.com/") self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) # make sure urls are converted to str - resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8') + resp = self.response_class(url="http://www.example.com/", encoding='utf-8') assert isinstance(resp.url, str) - resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8') + resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8') self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1') + resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1') self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') - resp = self.response_class(u"http://www.example.com/price/\xa3", + resp = self.response_class("http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(u"http://www.example.com/price/\xa3", + resp = self.response_class("http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') def test_unicode_body(self): unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ' '\u0442\u0435\u043a\u0441\u0442') - self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body') + self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body') original_string = unicode_string.encode('cp1251') r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') @@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest): def test_encoding(self): r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) - r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") + r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3") r3 = self.response_class("http://www.example.com", body=b"\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") @@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r5._headers_encoding(), None) self._assert_response_encoding(r5, "utf-8") assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' - self._assert_response_values(r1, 'utf-8', u"\xa3") - self._assert_response_values(r2, 'utf-8', u"\xa3") - self._assert_response_values(r3, 'iso-8859-1', u"\xa3") - self._assert_response_values(r6, 'gb18030', u"\u2015") - self._assert_response_values(r7, 'gb18030', u"\u2015") + self._assert_response_values(r1, 'utf-8', "\xa3") + self._assert_response_values(r2, 'utf-8', "\xa3") + self._assert_response_values(r3, 'iso-8859-1', "\xa3") + self._assert_response_values(r6, 'gb18030', "\u2015") + self._assert_response_values(r7, 'gb18030', "\u2015") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3") + self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3") def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" @@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=UKNOWN"]}, body=b"\xc2\xa3") self.assertEqual(r._declared_encoding(), None) - self._assert_response_values(r, 'utf-8', u"\xa3") + self._assert_response_values(r, 'utf-8', "\xa3") def test_utf16(self): """Test utf-16 because UnicodeDammit is known to have problems with""" r = self.response_class("http://www.example.com", body=b'\xff\xfeh\x00i\x00', encoding='utf-16') - self._assert_response_values(r, 'utf-16', u"hi") + self._assert_response_values(r, 'utf-16', "hi") def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): r6 = self.response_class("http://www.example.com", @@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest): body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') self.assertIn(r6.text, { - u'WORD\ufffd\ufffd', # w3lib < 1.19.0 - u'WORD\ufffd', # w3lib >= 1.19.0 + 'WORD\ufffd\ufffd', # w3lib < 1.19.0 + 'WORD\ufffd', # w3lib >= 1.19.0 }) def test_bom_is_removed_from_body(self): @@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest): # Test response without content-type and BOM encoding response = self.response_class(url, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') response = self.response_class(url, body=body) - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') self.assertEqual(response.encoding, 'utf-8') # Body caching sideeffect isn't triggered when encoding is declared in @@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest): # body response = self.response_class(url, headers=headers, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') self.assertEqual(response.encoding, 'utf-8') def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX') # XXX: Policy for replacing invalid chars may suffer minor variations - # but it should always contain the unicode replacement char (u'\ufffd') - assert u'\ufffd' in r.text, repr(r.text) - assert u'PREFIX' in r.text, repr(r.text) - assert u'SUFFIX' in r.text, repr(r.text) + # but it should always contain the unicode replacement char ('\ufffd') + assert '\ufffd' in r.text, repr(r.text) + assert 'PREFIX' in r.text, repr(r.text) + assert 'SUFFIX' in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs r = self.response_class("http://example.com", encoding='utf-8', body=b'\xf0value') - assert u'value' in r.text, repr(r.text) + assert 'value' in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') - # assert u'\ufffd' in r.text, repr(r.text) + # assert '\ufffd' in r.text, repr(r.text) def test_selector(self): body = b"Some page" @@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest): self.assertEqual( response.selector.xpath("//title/text()").getall(), - [u'Some page'] + ['Some page'] ) self.assertEqual( response.selector.css("title::text").getall(), - [u'Some page'] + ['Some page'] ) self.assertEqual( response.selector.re("Some (.*)"), - [u'page'] + ['page'] ) def test_selector_shortcuts(self): @@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( 'http://example.com', encoding='utf8', - body=u'click me'.encode('utf8') + body='click me'.encode('utf8') ) req = self._assert_followed_url( resp1.css('a')[0], @@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest): resp2 = self.response_class( 'http://example.com', encoding='cp1251', - body=u'click me'.encode('cp1251') + body='click me'.encode('cp1251') ) req = self._assert_followed_url( resp2.css('a')[0], @@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest): def test_body_as_unicode_deprecation_warning(self): with catch_warnings(record=True) as warnings: - r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8') - self.assertEqual(r1.body_as_unicode(), u'Hello') + r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8') + self.assertEqual(r1.body_as_unicode(), 'Hello') self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) @@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest): self.assertEqual( response.selector.xpath("//elem/text()").getall(), - [u'value'] + ['value'] ) def test_selector_shortcuts(self): diff --git a/tests/test_item.py b/tests/test_item.py index 60468971c..78d204e34 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,4 +1,3 @@ -import sys import unittest from unittest import mock from warnings import catch_warnings @@ -7,9 +6,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta -PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6) - - class ItemTest(unittest.TestCase): def assertSortedEqual(self, first, second, msg=None): @@ -20,8 +16,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - i['name'] = u'name' - self.assertEqual(i['name'], u'name') + i['name'] = 'name' + self.assertEqual(i['name'], 'name') def test_init(self): class TestItem(Item): @@ -30,17 +26,17 @@ class ItemTest(unittest.TestCase): i = TestItem() self.assertRaises(KeyError, i.__getitem__, 'name') - i2 = TestItem(name=u'john doe') - self.assertEqual(i2['name'], u'john doe') + i2 = TestItem(name='john doe') + self.assertEqual(i2['name'], 'john doe') - i3 = TestItem({'name': u'john doe'}) - self.assertEqual(i3['name'], u'john doe') + i3 = TestItem({'name': 'john doe'}) + self.assertEqual(i3['name'], 'john doe') i4 = TestItem(i3) - self.assertEqual(i4['name'], u'john doe') + self.assertEqual(i4['name'], 'john doe') - self.assertRaises(KeyError, TestItem, {'name': u'john doe', - 'other': u'foo'}) + self.assertRaises(KeyError, TestItem, {'name': 'john doe', + 'other': 'foo'}) def test_invalid_field(self): class TestItem(Item): @@ -56,7 +52,7 @@ class ItemTest(unittest.TestCase): number = Field() i = TestItem() - i['name'] = u'John Doe' + i['name'] = 'John Doe' i['number'] = 123 itemrepr = repr(i) @@ -101,9 +97,9 @@ class ItemTest(unittest.TestCase): i = TestItem() self.assertRaises(KeyError, i.get_name) - i['name'] = u'lala' - self.assertEqual(i.get_name(), u'lala') - i.change_name(u'other') + i['name'] = 'lala' + self.assertEqual(i.get_name(), 'lala') + i.change_name('other') self.assertEqual(i.get_name(), 'other') def test_metaclass(self): @@ -113,22 +109,22 @@ class ItemTest(unittest.TestCase): values = Field() i = TestItem() - i['name'] = u'John' + i['name'] = 'John' self.assertEqual(list(i.keys()), ['name']) self.assertEqual(list(i.values()), ['John']) - i['keys'] = u'Keys' - i['values'] = u'Values' + i['keys'] = 'Keys' + i['values'] = 'Values' self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name']) - self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John']) + self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John']) def test_metaclass_with_fields_attribute(self): class TestItem(Item): fields = {'new': Field(default='X')} - item = TestItem(new=u'New') + item = TestItem(new='New') self.assertSortedEqual(list(item.keys()), ['new']) - self.assertSortedEqual(list(item.values()), [u'New']) + self.assertSortedEqual(list(item.values()), ['New']) def test_metaclass_inheritance(self): class ParentItem(Item): @@ -238,8 +234,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - i['name'] = u'John' - self.assertEqual(dict(i), {'name': u'John'}) + i['name'] = 'John' + self.assertEqual(dict(i), {'name': 'John'}) def test_copy(self): class TestItem(Item): @@ -280,14 +276,6 @@ class ItemMetaTest(unittest.TestCase): with mock.patch.object(base, '__new__', new_mock): class MyItem(Item): - if not PY36_PLUS: - # This attribute is an internal attribute in Python 3.6+ - # and must be propagated properly. See - # https://docs.python.org/3.6/reference/datamodel.html#creating-the-class-object - # In <3.6, we add a dummy attribute just to ensure the - # __new__ method propagates it correctly. - __classcell__ = object() - def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 @@ -312,7 +300,7 @@ class ItemMetaClassCellRegression(unittest.TestCase): # requirement. When not done properly raises an error: # TypeError: __class__ set to # defining 'MyItem' as - super(MyItem, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class DictItemTest(unittest.TestCase): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 8d4538eed..6f133d77a 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -31,31 +31,31 @@ class Base: page4_url = 'http://example.com/page%204.html' self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) def test_extract_filter_allow(self): lx = self.extractor_cls(allow=('sample', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=('sample', ), unique=False) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) @@ -63,10 +63,10 @@ class Base: lx = self.extractor_cls(allow=('sample', ), unique=False, canonicalize=True) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment') ]) @@ -74,22 +74,22 @@ class Base: lx = self.extractor_cls(allow=('sample',), unique=True, canonicalize=True) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), ]) def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=('sample', ), deny=('3', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=('google.com', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) def test_extraction_using_single_values(self): @@ -97,27 +97,27 @@ class Base: lx = self.extractor_cls(allow='sample') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) lx = self.extractor_cls(allow='sample', deny='3') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(allow_domains='google.com') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) lx = self.extractor_cls(deny_domains='example.com') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) def test_nofollow(self): @@ -145,11 +145,11 @@ class Base: lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/about.html', text=u'About us'), - Link(url='http://example.org/follow.html', text=u'Follow this link'), - Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True), - Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://example.org/about.html', text='About us'), + Link(url='http://example.org/follow.html', text='Follow this link'), + Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True), + Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ]) def test_matches(self): @@ -183,8 +183,8 @@ class Base: def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) def test_restrict_xpaths_encoding(self): @@ -202,14 +202,14 @@ class Base: lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/about.html', text=u'About us\xa3')]) + [Link(url='http://example.org/about.html', text='About us\xa3')]) def test_restrict_xpaths_with_html_entities(self): html = b'

    text

    ' response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15') links = self.extractor_cls(restrict_xpaths='//p').extract_links(response) self.assertEqual(links, - [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')]) + [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')]) def test_restrict_xpaths_concat_in_handle_data(self): """html entities cause SGMLParser to call handle_data hook twice""" @@ -217,22 +217,22 @@ class Base: response = HtmlResponse("http://example.org", body=body, encoding='gb18030') lx = self.extractor_cls(restrict_xpaths="//div") self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', + [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c', fragment='', nofollow=False)]) def test_restrict_css(self): lx = self.extractor_cls(restrict_css=('#subwrapper a',)) self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2') + Link(url='http://example.com/sample2.html', text='sample 2') ]) def test_restrict_css_and_restrict_xpaths_together(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ), restrict_css=('#subwrapper + a', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), ]) def test_area_tag_with_unicode_present(self): @@ -243,7 +243,7 @@ class Base: lx.extract_links(response) lx.extract_links(response) self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text=u'', + [Link(url='http://example.org/foo', text='', fragment='', nofollow=False)]) def test_encoded_url(self): @@ -251,7 +251,7 @@ class Base: response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), + Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False), ]) def test_encoded_url_in_restricted_xpath(self): @@ -259,7 +259,7 @@ class Base: response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') lx = self.extractor_cls(restrict_xpaths="//div") self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), + Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False), ]) def test_ignored_extensions(self): @@ -268,7 +268,7 @@ class Base: response = HtmlResponse("http://example.org/", body=html) lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text=u'asd'), + Link(url='http://example.org/page.html', text='asd'), ]) # override denied extensions @@ -308,25 +308,25 @@ class Base: page4_url = 'http://example.com/page%204.html' self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=()) self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample2.jpg', text=u''), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample2.jpg', text=''), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) lx = self.extractor_cls(attrs=None) @@ -344,24 +344,24 @@ class Base: lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(tags="area") self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample1.html', text=''), ]) lx = self.extractor_cls(tags="a") self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=()) self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample2.jpg', text=u''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample2.jpg', text=''), ]) def test_tags_attrs(self): @@ -375,14 +375,14 @@ class Base: lx = self.extractor_cls(tags='div', attrs='data-url') self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False) ]) lx = self.extractor_cls(tags=('div',), attrs=('data-url',)) self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False) ]) def test_xhtml(self): @@ -420,13 +420,13 @@ class Base: self.assertEqual( lx.extract_links(response), [ - Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', + Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text='Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', + Link(url='http://example.com/nofollow2.html', text='Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ] ) @@ -436,13 +436,13 @@ class Base: self.assertEqual( lx.extract_links(response), [ - Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', + Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text='Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', + Link(url='http://example.com/nofollow2.html', text='Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ] ) @@ -455,8 +455,8 @@ class Base: response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + Link(url='http://example.org/item1.html', text='Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text='Item 3', nofollow=False), ]) def test_ftp_links(self): @@ -467,7 +467,7 @@ class Base: response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8') lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), + Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False), ]) def test_pickle_extractor(self): @@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + Link(url='http://example.org/item1.html', text='Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text='Item 3', nofollow=False), ]) def test_link_restrict_text(self): @@ -501,22 +501,22 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Simple text inclusion test lx = self.extractor_cls(restrict_text='dog') self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) # Unique regex test lx = self.extractor_cls(restrict_text=r'of.*dog') self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) # Multiple regex test lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat']) self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False), - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) def test_restrict_xpaths_with_html_entities(self): - super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() + super().test_restrict_xpaths_with_html_entities() def test_filteringlinkextractor_deprecation_warning(self): """Make sure the FilteringLinkExtractor deprecation warning is not diff --git a/tests/test_loader.py b/tests/test_loader.py index 581183625..b0bc82f4e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase): def test_add_value_on_unknown_field(self): il = TestItemLoader() - self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) + self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo']) def test_load_item_using_default_loader(self): i = TestItem() - i['summary'] = u'lala' + i['summary'] = 'lala' il = ItemLoader(item=i) - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() assert item is i - self.assertEqual(item['summary'], [u'lala']) - self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['summary'], ['lala']) + self.assertEqual(item['name'], ['marta']) def test_load_item_using_custom_loader(self): il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() - self.assertEqual(item['name'], [u'Marta']) + self.assertEqual(item['name'], ['Marta']) class InitializationTestMixin: @@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase): temp = Field() def __init__(self, *args, **kwargs): - super(TempItem, self).__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) self.setdefault('temp', 0.3) class TempLoader(ItemLoader): @@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertRaises(RuntimeError, l.get_css, '#name::text') def test_init_method_with_selector(self): - sel = Selector(text=u"
    marta
    ") + sel = Selector(text="
    marta
    ") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_selector_css(self): - sel = Selector(text=u"
    marta
    ") + sel = Selector(text="
    marta
    ") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_response(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_response_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.add_css('url', 'a::attr(href)') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) # combining/accumulating CSS selectors and XPath expressions l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta']) l.add_xpath('url', '//img/@src') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png']) def test_add_xpath_re(self): l = TestItemLoader(response=self.response) l.add_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) def test_replace_xpath(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath('name', '//p/text()') - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.replace_xpath('name', ['//p/text()', '//div/text()']) - self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta']) + self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta']) def test_get_xpath(self): l = TestItemLoader(response=self.response) - self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph']) - self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph') - self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa') + self.assertEqual(l.get_xpath('//p/text()'), ['paragraph']) + self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph') + self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa') - self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta']) + self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta']) def test_replace_xpath_multi_fields(self): l = TestItemLoader(response=self.response) l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) def test_replace_xpath_re(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) def test_add_css_re(self): l = TestItemLoader(response=self.response) l.add_css('name', 'div::text', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) l.add_css('url', 'a::attr(href)', re='http://(.+)') - self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['www.scrapy.org']) def test_replace_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_css('name', 'p::text') - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.replace_css('name', ['p::text', 'div::text']) - self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta']) + self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta']) l.add_css('url', 'a::attr(href)', re='http://(.+)') - self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['www.scrapy.org']) l.replace_css('url', 'img::attr(src)') - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) def test_get_css(self): l = TestItemLoader(response=self.response) - self.assertEqual(l.get_css('p::text'), [u'paragraph']) - self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph') - self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa') + self.assertEqual(l.get_css('p::text'), ['paragraph']) + self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph') + self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa') - self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta']) + self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta']) self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']), - [u'http://www.scrapy.org', u'/images/logo.png']) + ['http://www.scrapy.org', '/images/logo.png']) def test_replace_css_multi_fields(self): l = TestItemLoader(response=self.response) l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x}) - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x}) - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) def test_replace_css_re(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('url', 'a::attr(href)') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)') - self.assertEqual(l.get_output_value('url'), [u'scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['scrapy.org']) class SubselectorLoaderTest(unittest.TestCase): @@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css('name_div', '#id') nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value('name'), [u'marta']) - self.assertEqual(l.get_output_value('name_div'), [u'
    marta
    ']) - self.assertEqual(l.get_output_value('name_value'), [u'marta']) + self.assertEqual(l.get_output_value('name'), ['marta']) + self.assertEqual(l.get_output_value('name_div'), ['
    marta
    ']) + self.assertEqual(l.get_output_value('name_value'), ['marta']) self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) @@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css('name_div', '#id') nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value('name'), [u'marta']) - self.assertEqual(l.get_output_value('name_div'), [u'
    marta
    ']) - self.assertEqual(l.get_output_value('name_value'), [u'marta']) + self.assertEqual(l.get_output_value('name'), ['marta']) + self.assertEqual(l.get_output_value('name_div'), ['
    marta
    ']) + self.assertEqual(l.get_output_value('name_value'), ['marta']) self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) @@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase): nl2 = nl1.nested_xpath('a') l.add_xpath('url', '//footer/a/@href') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) nl1.replace_xpath('url', 'img/@src') - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) nl2.replace_xpath('url', '@href') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) def test_nested_ordering(self): l = NestedItemLoader(response=self.response) @@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase): l.add_xpath('url', '//footer/a/@href') self.assertEqual(l.get_output_value('url'), [ - u'/images/logo.png', - u'http://www.scrapy.org', - u'homepage', - u'http://www.scrapy.org', + '/images/logo.png', + 'http://www.scrapy.org', + 'homepage', + 'http://www.scrapy.org', ]) def test_nested_load_item(self): @@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase): assert item is nl1.item assert item is nl2.item - self.assertEqual(item['name'], [u'marta']) - self.assertEqual(item['url'], [u'http://www.scrapy.org']) - self.assertEqual(item['image'], [u'/images/logo.png']) + self.assertEqual(item['name'], ['marta']) + self.assertEqual(item['url'], ['http://www.scrapy.org']) + self.assertEqual(item['image'], ['/images/logo.png']) # Functions as processors diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index d0a59e8cd..41afa2896 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase): def test_load_item_using_default_loader(self): i = TestItem() - i['summary'] = u'lala' + i['summary'] = 'lala' il = ItemLoader(item=i) - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() assert item is i - self.assertEqual(item['summary'], [u'lala']) - self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['summary'], ['lala']) + self.assertEqual(item['name'], ['marta']) def test_load_item_using_custom_loader(self): il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() - self.assertEqual(item['name'], [u'Marta']) + self.assertEqual(item['name'], ['Marta']) def test_load_item_ignore_none_field_values(self): def validate_sku(value): @@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase): price_out = Compose(TakeFirst(), float) sku_out = Compose(TakeFirst(), validate_sku) - valid_fragment = u'SKU: 1234' - invalid_fragment = u'SKU: not available' + valid_fragment = 'SKU: 1234' + invalid_fragment = 'SKU: not available' sku_re = 'SKU: (.+)' il = MyLoader(item={}) # Should not return "sku: None". il.add_value('sku', [invalid_fragment], re=sku_re) # Should not ignore empty values. - il.add_value('name', u'') - il.add_value('price', [u'0']) + il.add_value('name', '') + il.add_value('price', ['0']) self.assertEqual(il.load_item(), { - 'name': u'', + 'name': '', 'price': 0.0, }) il.replace_value('sku', [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()['sku'], u'1234') + self.assertEqual(il.load_item()['sku'], '1234') def test_self_referencing_loader(self): class MyLoader(ItemLoader): @@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase): def test_add_value(self): il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.add_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe']) + il.add_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['Marta']) + self.assertEqual(il.get_output_value('name'), ['Marta']) + il.add_value('name', 'pepe') + self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe']) + self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe']) # test add object value il.add_value('summary', {'key': 1}) self.assertEqual(il.get_collected_values('summary'), [{'key': 1}]) - il.add_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim']) + il.add_value(None, 'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim']) def test_add_zero(self): il = NameItemLoader() @@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase): def test_replace_value(self): il = TestItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.replace_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Pepe']) + il.replace_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['Marta']) + self.assertEqual(il.get_output_value('name'), ['Marta']) + il.replace_value('name', 'pepe') + self.assertEqual(il.get_collected_values('name'), ['Pepe']) + self.assertEqual(il.get_output_value('name'), ['Pepe']) - il.replace_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Jim']) + il.replace_value(None, 'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), ['Jim']) def test_get_value(self): il = NameItemLoader() - self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) - self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) - self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) + self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper)) + self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$')) + self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')) - il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$') - self.assertEqual([u'foo'], il.get_collected_values('name')) - il.replace_value('name', u'name:bar', re=u'name:(.*)$') - self.assertEqual([u'bar'], il.get_collected_values('name')) + il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$') + self.assertEqual(['foo'], il.get_collected_values('name')) + il.replace_value('name', 'name:bar', re='name:(.*)$') + self.assertEqual(['bar'], il.get_collected_values('name')) def test_iter_on_input_processor_input(self): class NameFirstItemLoader(NameItemLoader): name_in = TakeFirst() il = NameFirstItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.add_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.add_value('name', ['marta', 'jose']) + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.replace_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.replace_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.replace_value('name', ['marta', 'jose']) + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.add_value('name', u'marta') - il.add_value('name', [u'jose', u'pedro']) - self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose']) + il.add_value('name', 'marta') + il.add_value('name', ['jose', 'pedro']) + self.assertEqual(il.get_collected_values('name'), ['marta', 'jose']) def test_map_compose_filter(self): def filter_world(x): @@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase): name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Mart']) item = il.load_item() - self.assertEqual(item['name'], [u'Mart']) + self.assertEqual(item['name'], ['Mart']) def test_default_input_processor(self): il = DefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mart']) def test_inherited_default_input_processor(self): class InheritDefaultedItemLoader(DefaultedItemLoader): pass il = InheritDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mart']) def test_input_processor_inheritance(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(lambda v: v.lower()) il = ChildItemLoader() - il.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.add_value('url', 'HTTP://scrapy.ORG') + self.assertEqual(il.get_output_value('url'), ['http://scrapy.org']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Marta']) class ChildChildItemLoader(ChildItemLoader): url_in = MapCompose(lambda v: v.upper()) summary_in = MapCompose(lambda v: v) il = ChildChildItemLoader() - il.add_value('url', u'http://scrapy.org') - self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.add_value('url', 'http://scrapy.org') + self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Marta']) def test_empty_map_compose(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = MapCompose() il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['marta']) def test_identity_input_processor(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = Identity() il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['marta']) def test_extend_custom_input_processors(self): class ChildItemLoader(TestItemLoader): name_in = MapCompose(TestItemLoader.name_in, str.swapcase) il = ChildItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mARTA']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mARTA']) def test_extend_default_input_processors(self): class ChildDefaultedItemLoader(DefaultedItemLoader): name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) il = ChildDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'MART']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['MART']) def test_output_processor_using_function(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class TakeFirstItemLoader(TestItemLoader): - name_out = u" ".join + name_out = " ".join il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar Ta') def test_output_processor_error(self): class TestItemLoader(ItemLoader): @@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = MapCompose(float) il = TestItemLoader() - il.add_value('name', [u'$10']) + il.add_value('name', ['$10']) try: - float(u'$10') + float('$10') except Exception as e: expected_exc_str = str(e) @@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase): def test_output_processor_using_classes(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class TakeFirstItemLoader(TestItemLoader): name_out = Join() il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar Ta') class TakeFirstItemLoader(TestItemLoader): name_out = Join("
    ") il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar
    Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar
    Ta') def test_default_output_processor(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class LalaItemLoader(TestItemLoader): default_output_processor = Identity() il = LalaItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) def test_loader_context_on_declaration(self): class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args, key=u'val') + url_in = MapCompose(processor_with_args, key='val') il = ChildItemLoader() - il.add_value('url', u'text') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_loader_context_on_instantiation(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(processor_with_args) - il = ChildItemLoader(key=u'val') - il.add_value('url', u'text') + il = ChildItemLoader(key='val') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_loader_context_on_assign(self): @@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase): url_in = MapCompose(processor_with_args) il = ChildItemLoader() - il.context['key'] = u'val' - il.add_value('url', u'text') + il.context['key'] = 'val' + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_item_passed_to_input_processor_functions(self): @@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase): it = TestItem(name='marta') il = ChildItemLoader(item=it) - il.add_value('url', u'text') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['marta']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['marta']) def test_compose_processor(self): @@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() - il.add_value('name', [u'marta', u'other']) - self.assertEqual(il.get_output_value('name'), u'Mart') + il.add_value('name', ['marta', 'other']) + self.assertEqual(il.get_output_value('name'), 'Mart') item = il.load_item() - self.assertEqual(item['name'], u'Mart') + self.assertEqual(item['name'], 'Mart') def test_partial_processor(self): def join(values, sep=None, loader_context=None, ignored=None): @@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase): summary_out = Compose(partial(join, ignored='foo')) il = TestItemLoader() - il.add_value('name', [u'rabbit', u'hole']) - il.add_value('url', [u'rabbit', u'hole']) - il.add_value('summary', [u'rabbit', u'hole']) + il.add_value('name', ['rabbit', 'hole']) + il.add_value('url', ['rabbit', 'hole']) + il.add_value('summary', ['rabbit', 'hole']) item = il.load_item() - self.assertEqual(item['name'], u'rabbit+hole') - self.assertEqual(item['url'], u'rabbit.hole') - self.assertEqual(item['summary'], u'rabbithole') + self.assertEqual(item['name'], 'rabbit+hole') + self.assertEqual(item['url'], 'rabbit.hole') + self.assertEqual(item['summary'], 'rabbithole') def test_error_input_processor(self): class TestItem(Item): @@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other']) + ['marta', 'other']) def test_error_output_processor(self): class TestItem(Item): @@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = Compose(Join(), float) il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') with self.assertRaises(ValueError): il.load_item() @@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other'], Compose(float)) + ['marta', 'other'], Compose(float)) class InitializationFromDictTest(unittest.TestCase): @@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase): class TempDict(dict): def __init__(self, *args, **kwargs): - super(TempDict, self).__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) self.setdefault('temp', 0.3) class TempLoader(ItemLoader): @@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase): def test_join(self): proc = Join() self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) - self.assertEqual(proc(['', 'hello', 'world']), u' hello world') - self.assertEqual(proc(['hello', 'world']), u'hello world') + self.assertEqual(proc(['', 'hello', 'world']), ' hello world') + self.assertEqual(proc(['hello', 'world']), 'hello world') self.assertIsInstance(proc(['hello', 'world']), str) def test_compose(self): @@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase): def filter_world(x): return None if x == 'world' else x proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), - [u'HELLO', u'THIS', u'IS', u'SCRAPY']) + self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), + ['HELLO', 'THIS', 'IS', 'SCRAPY']) proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) proc = MapCompose(filter_world, str.upper) @@ -657,7 +657,7 @@ class SelectJmesTestCase(unittest.TestCase): self.assertEqual( test, expected, - msg='test "{}" got {} expected {}'.format(tl, test, expected) + msg=f'test "{tl}" got {test} expected {expected}' ) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 7064337ad..dc5be398f 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -20,7 +20,7 @@ class CustomItem(Item): name = Field() def __str__(self): - return "name: %s" % self['name'] + return f"name: {self['name']}" class LogFormatterTestCase(unittest.TestCase): @@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase): def test_dropped(self): item = {} - exception = Exception(u"\u2018") + exception = Exception("\u2018") response = Response("http://www.example.com") logkws = self.formatter.dropped(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, [u"Dropped: \u2018", '{}']) + self.assertEqual(lines, ["Dropped: \u2018", '{}']) def test_item_error(self): # In practice, the complete traceback is shown by passing the @@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com") logkws = self.formatter.item_error(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, u"Error processing {'key': 'value'}") + self.assertEqual(logline, "Error processing {'key': 'value'}") def test_spider_error(self): # In practice, the complete traceback is shown by passing the @@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase): def test_scraped(self): item = CustomItem() - item['name'] = u'\xa3' + item['name'] = '\xa3' response = Response("http://www.example.com") logkws = self.formatter.scraped(item, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3']) + self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3']) class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): - kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider) + kwargs = super().crawled(request, response, spider) CRAWLEDMSG = ( - u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" + "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" ) log_args = kwargs['args'] log_args['flags'] = str(request.flags) diff --git a/tests/test_mail.py b/tests/test_mail.py index 53dbc0686..9b248fbfa 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase): self.catched_msg = dict(**kwargs) def test_send_utf8(self): - subject = u'sübjèçt' - body = u'bödÿ-àéïöñß' + subject = 'sübjèçt' + body = 'bödÿ-àéïöñß' mailsender = MailSender(debug=True) mailsender.send(to=['test@scrapy.org'], subject=subject, body=body, charset='utf-8', _callback=self._catch_mail_sent) @@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"') def test_send_attach_utf8(self): - subject = u'sübjèçt' - body = u'bödÿ-àéïöñß' + subject = 'sübjèçt' + body = 'bödÿ-àéïöñß' attach = BytesIO() attach.write(body.encode('utf-8')) attach.seek(0) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3364d2258..8651431b5 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -50,10 +50,10 @@ class TestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] + return [M1, MOff, M3] def _add_middleware(self, mw): - super(TestMiddlewareManager, self)._add_middleware(mw) + super()._add_middleware(mw) if hasattr(mw, 'process'): self.methods['process'].append(mw.process) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 9af5affec..55fcfa7ba 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -123,10 +123,10 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4) self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1) - self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3) + self.assertEqual(crawler.stats.get_value(f'downloader/response_status_count/{code}'), 3) # check that logs do show the failure on the file downloads - file_dl_failure = 'File (code: %d): Error downloading file from' % code + file_dl_failure = f'File (code: {code}): Error downloading file from' self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index a023dfcc8..1dd7031fe 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -161,6 +161,19 @@ class FilesPipelineTestCase(unittest.TestCase): for p in patchers: p.stop() + def test_file_path_from_item(self): + """ + Custom file path based on item data, overriding default implementation + """ + class CustomFilesPipeline(FilesPipeline): + def file_path(self, request, response=None, info=None, item=None): + return f'full/{item.get("path")}' + + file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(file_path(request, item=item), 'full/path-to-store-file') + class FilesPipelineTestCaseFieldsMixin: @@ -482,7 +495,7 @@ class TestFTPFileStore(unittest.TestCase): self.assertIn('last_modified', stat) self.assertIn('checksum', stat) self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6') - path = '%s/%s' % (store.basedir, path) + path = f'{store.basedir}/{path}' content = get_ftp_content_and_delete( path, store.host, store.port, store.username, store.password, store.USE_ACTIVE_MODE) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 082e9ee21..ad138a2dc 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -128,11 +128,11 @@ class DeprecatedImagesPipeline(ImagesPipeline): def image_key(self, url): image_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return 'empty/%s.jpg' % (image_guid) + return f'empty/{image_guid}.jpg' def thumb_key(self, url, thumb_id): thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid) + return f'thumbsup/{thumb_id}/{thumb_guid}.jpg' class ImagesPipelineTestCaseFieldsMixin: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 19ff00350..6afd47497 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -7,7 +7,9 @@ from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.request import request_fingerprint +from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.pipelines.files import FileException from scrapy.utils.log import failure_to_exc_info @@ -162,34 +164,34 @@ class BaseMediaPipelineTestCase(unittest.TestCase): class MockedMediaPipeline(MediaPipeline): def __init__(self, *args, **kwargs): - super(MockedMediaPipeline, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._mockcalled = [] def download(self, request, info): self._mockcalled.append('download') - return super(MockedMediaPipeline, self).download(request, info) + return super().download(request, info) - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): self._mockcalled.append('media_to_download') if 'result' in request.meta: return request.meta.get('result') - return super(MockedMediaPipeline, self).media_to_download(request, info) + return super().media_to_download(request, info) def get_media_requests(self, item, info): self._mockcalled.append('get_media_requests') return item.get('requests') - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): self._mockcalled.append('media_downloaded') - return super(MockedMediaPipeline, self).media_downloaded(response, request, info) + return super().media_downloaded(response, request, info) def media_failed(self, failure, request, info): self._mockcalled.append('media_failed') - return super(MockedMediaPipeline, self).media_failed(failure, request, info) + return super().media_failed(failure, request, info) def item_completed(self, results, item, info): self._mockcalled.append('item_completed') - item = super(MockedMediaPipeline, self).item_completed(results, item, info) + item = super().item_completed(results, item, info) item['results'] = results return item @@ -335,6 +337,123 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ['get_media_requests', 'media_to_download', 'item_completed']) +class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): + + def __init__(self, *args, **kwargs): + super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs) + self._mockcalled = [] + + def get_media_requests(self, item, info): + item_url = item['image_urls'][0] + return Request( + item_url, + meta={'response': Response(item_url, status=200, body=b'data')} + ) + + def inc_stats(self, *args, **kwargs): + return True + + def media_to_download(self, request, info): + self._mockcalled.append('media_to_download') + return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info) + + def media_downloaded(self, response, request, info): + self._mockcalled.append('media_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info) + + def file_downloaded(self, response, request, info): + self._mockcalled.append('file_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info) + + def file_path(self, request, response=None, info=None): + self._mockcalled.append('file_path') + return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info) + + def get_images(self, response, request, info): + self._mockcalled.append('get_images') + return [] + + def image_downloaded(self, response, request, info): + self._mockcalled.append('image_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info) + + +class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): + + def setUp(self): + self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) + self.pipe.open_spider(None) + self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[]) + + def _assert_method_called_with_warnings(self, method, message, warnings): + self.assertIn(method, self.pipe._mockcalled) + warningShown = False + for warning in warnings: + if warning['message'] == message and warning['category'] == ScrapyDeprecationWarning: + warningShown = True + self.assertTrue(warningShown) + + @inlineCallbacks + def test_media_to_download_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'media_to_download(self, request, info) is deprecated, ' + 'please use media_to_download(self, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('media_to_download', message, warnings) + + @inlineCallbacks + def test_media_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'media_downloaded(self, response, request, info) is deprecated, ' + 'please use media_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('media_downloaded', message, warnings) + + @inlineCallbacks + def test_file_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'file_downloaded(self, response, request, info) is deprecated, ' + 'please use file_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('file_downloaded', message, warnings) + + @inlineCallbacks + def test_file_path_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'file_path(self, request, response=None, info=None) is deprecated, ' + 'please use file_path(self, request, response=None, info=None, *, item=None)' + ) + self._assert_method_called_with_warnings('file_path', message, warnings) + + @inlineCallbacks + def test_get_images_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'get_images(self, response, request, info) is deprecated, ' + 'please use get_images(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('get_images', message, warnings) + + @inlineCallbacks + def test_image_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'image_downloaded(self, response, request, info) is deprecated, ' + 'please use image_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('image_downloaded', message, warnings) + + class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): def _assert_request_no3xx(self, pipeline_class, settings): diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index a56e3c39a..9eabe6b49 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -7,7 +7,6 @@ from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit from unittest import skipIf -import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -37,14 +36,14 @@ sys.exit(mitmdump()) '-c', script, '--listen-host', '127.0.0.1', '--listen-port', '0', - '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), + '--proxyauth', f'{self.auth_user}:{self.auth_pass}', '--certs', cert_path, '--ssl-insecure', ], stdout=PIPE, env=get_testenv()) line = self.proc.stdout.readline().decode('utf-8') host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1) - address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port) + address = f'http://{self.auth_user}:{self.auth_pass}@{host_port}' return address def stop(self): @@ -58,8 +57,6 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -@skipIf(sys.version_info < (3, 5, 4), - "requires mitmproxy < 3.0.0, which these tests do not support") @skipIf("pypy" in sys.executable, "mitmproxy does not support PyPy") @skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), @@ -88,14 +85,6 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, log) - @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info >= (3, 6)) - @defer.inlineCallbacks - def test_https_connect_tunnel_error(self): - crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(log) - @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) @@ -118,7 +107,7 @@ class ProxyConnectTestCase(TestCase): def _assert_got_response_code(self, code, log): print(log) - self.assertEqual(str(log).count('Crawled (%d)' % code), 1) + self.assertEqual(str(log).count(f'Crawled ({code})'), 1) def _assert_got_tunnel_error(self, log): print(log) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py new file mode 100644 index 000000000..907117468 --- /dev/null +++ b/tests/test_request_attribute_binding.py @@ -0,0 +1,202 @@ +from twisted.internet import defer +from twisted.trial.unittest import TestCase + +from scrapy import Request, signals +from scrapy.crawler import CrawlerRunner +from scrapy.http.response import Response + +from testfixtures import LogCapture + +from tests.mockserver import MockServer +from tests.spiders import SingleRequestSpider + + +OVERRIDEN_URL = "https://example.org" + + +class ProcessResponseMiddleware: + def process_response(self, request, response, spider): + return response.replace(request=Request(OVERRIDEN_URL)) + + +class RaiseExceptionRequestMiddleware: + def process_request(self, request, spider): + 1 / 0 + return request + + +class CatchExceptionOverrideRequestMiddleware: + def process_exception(self, request, exception, spider): + return Response( + url="http://localhost/", + body=b"Caught " + exception.__class__.__name__.encode("utf-8"), + request=Request(OVERRIDEN_URL), + ) + + +class CatchExceptionDoNotOverrideRequestMiddleware: + def process_exception(self, request, exception, spider): + return Response( + url="http://localhost/", + body=b"Caught " + exception.__class__.__name__.encode("utf-8"), + ) + + +class AlternativeCallbacksSpider(SingleRequestSpider): + name = "alternative_callbacks_spider" + + def alt_callback(self, response, foo=None): + self.logger.info("alt_callback was invoked with foo=%s", foo) + + +class AlternativeCallbacksMiddleware: + def process_response(self, request, response, spider): + new_request = request.replace( + url=OVERRIDEN_URL, + callback=spider.alt_callback, + cb_kwargs={"foo": "bar"}, + ) + return response.replace(request=new_request) + + +class CrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_response_200(self): + url = self.mockserver.url("/status?n=200") + crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_response_error(self): + for status in ("404", "500"): + url = self.mockserver.url(f"/status?n={status}") + crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + failure = crawler.spider.meta["failure"] + response = failure.value.response + self.assertEqual(failure.request.url, url) + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_downloader_middleware_raise_exception(self): + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + failure = crawler.spider.meta["failure"] + self.assertEqual(failure.request.url, url) + self.assertIsInstance(failure.value, ZeroDivisionError) + + @defer.inlineCallbacks + def test_downloader_middleware_override_request_in_process_response(self): + """ + Downloader middleware which returns a response with an specific 'request' attribute. + + * The spider callback should receive the overriden response.request + * Handlers listening to the response_received signal should receive the overriden response.request + * The "crawled" log message should show the overriden response.request + """ + signal_params = {} + + def signal_handler(response, request, spider): + signal_params["response"] = response + signal_params["request"] = request + + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".ProcessResponseMiddleware": 595, + } + }) + crawler = runner.create_crawler(SingleRequestSpider) + crawler.signals.connect(signal_handler, signal=signals.response_received) + + with LogCapture() as log: + yield crawler.crawl(seed=url, mockserver=self.mockserver) + + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.request.url, OVERRIDEN_URL) + + self.assertEqual(signal_params["response"].url, url) + self.assertEqual(signal_params["request"].url, OVERRIDEN_URL) + + log.check_present( + ("scrapy.core.engine", "DEBUG", f"Crawled (200) (referer: None)"), + ) + + @defer.inlineCallbacks + def test_downloader_middleware_override_in_process_exception(self): + """ + An exception is raised but caught by the next middleware, which + returns a Response with a specific 'request' attribute. + + The spider callback should receive the overriden response.request + """ + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + __name__ + ".CatchExceptionOverrideRequestMiddleware": 595, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.body, b"Caught ZeroDivisionError") + self.assertEqual(response.request.url, OVERRIDEN_URL) + + @defer.inlineCallbacks + def test_downloader_middleware_do_not_override_in_process_exception(self): + """ + An exception is raised but caught by the next middleware, which + returns a Response without a specific 'request' attribute. + + The spider callback should receive the original response.request + """ + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.body, b"Caught ZeroDivisionError") + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_downloader_middleware_alternative_callback(self): + """ + Downloader middleware which returns a response with a + specific 'request' attribute, with an alternative callback + """ + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".AlternativeCallbacksMiddleware": 595, + } + }) + crawler = runner.create_crawler(AlternativeCallbacksSpider) + + with LogCapture() as log: + url = self.mockserver.url("/status?n=200") + yield crawler.crawl(seed=url, mockserver=self.mockserver) + + log.check_present( + ("alternative_callbacks_spider", "INFO", "alt_callback was invoked with foo=bar"), + ) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 5cfef8e7d..373b2e49c 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider): name = 'signal_catcher' def __init__(self, crawler, url, *args, **kwargs): - super(SignalCatcherSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index dd19a69d5..c07d3a99c 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -17,22 +17,22 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_filename(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_content_disposition(self): mappings = [ (b'attachment; filename="data.xml"', XmlResponse), (b'attachment; filename=data.xml', XmlResponse), - (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response), - (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response), - (u'attachment;filename=data高.doc'.encode('gbk'), Response), - (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse), - (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse), + ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response), + ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response), + ('attachment;filename=data高.doc'.encode('gbk'), Response), + ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse), + ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse), ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_content_type(self): mappings = [ @@ -47,7 +47,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_body(self): mappings = [ @@ -58,7 +58,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_body(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_headers(self): mappings = [ @@ -70,7 +70,7 @@ class ResponseTypesTest(unittest.TestCase): for source, cls in mappings: source = Headers(source) retcls = responsetypes.from_headers(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_args(self): # TODO: add more tests that check precedence between the different arguments @@ -86,7 +86,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_args(**source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_custom_mime_types_loaded(self): # check that mime.types files shipped with scrapy are loaded diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 24aaaf7ec..4b15d0fab 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -93,7 +93,7 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) def test_unicode_url_and_useragent(self): - robotstxt_robotstxt_body = u""" + robotstxt_robotstxt_body = """ User-Agent: * Disallow: /admin/ Disallow: /static/ @@ -107,17 +107,17 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://site.local/", "*")) self.assertFalse(rp.allowed("https://site.local/admin/", "*")) self.assertFalse(rp.allowed("https://site.local/static/", "*")) - self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt")) + self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt")) self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*")) - self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*")) + self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*")) self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*")) - self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt")) + self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt")) class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import PythonRobotParser - super(PythonRobotParserTest, self)._setUp(PythonRobotParser) + super()._setUp(PythonRobotParser) def test_length_based_precedence(self): raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.") @@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import ReppyRobotParser - super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser) + super()._setUp(ReppyRobotParser) def test_order_based_precedence(self): raise unittest.SkipTest("Reppy does not support order based directives precedence.") @@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import RerpRobotParser - super(RerpRobotParserTest, self)._setUp(RerpRobotParser) + super()._setUp(RerpRobotParser) def test_length_based_precedence(self): raise unittest.SkipTest("Rerp does not support length based directives precedence.") @@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import ProtegoRobotParser - super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser) + super()._setUp(ProtegoRobotParser) def test_order_based_precedence(self): raise unittest.SkipTest("Protego does not support order based directives precedence.") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 2b6cb0902..512a7460e 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -53,7 +53,7 @@ class MockCrawler(Crawler): JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', ) - super(MockCrawler, self).__init__(Spider, settings) + super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) @@ -296,7 +296,7 @@ class StartUrlsSpider(Spider): def __init__(self, start_urls): self.start_urls = start_urls - super(StartUrlsSpider, self).__init__(name='StartUrlsSpider') + super().__init__(name='StartUrlsSpider') def parse(self, response): pass diff --git a/tests/test_selector.py b/tests/test_selector.py index 00e663c11..cff8d0393 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase): ) self.assertEqual( [x.get() for x in sel.xpath("//input[@name='a']/@name")], - [u'a'] + ['a'] ) self.assertEqual( [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - [u'12.0'] + ['12.0'] ) self.assertEqual( sel.xpath("concat('xpath', 'rules')").getall(), - [u'xpathrules'] + ['xpathrules'] ) self.assertEqual( [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - [u'12'] + ['12'] ) def test_root_base_url(self): @@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase): sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'xml') self.assertEqual(sel.xpath("//div").getall(), - [u'

    Hello

    ']) + ['

    Hello

    ']) sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'html') self.assertEqual(sel.xpath("//div").getall(), - [u'

    Hello

    ']) + ['

    Hello

    ']) def test_http_header_encoding_precedence(self): - # u'\xa3' = pound symbol in unicode - # u'\xc2\xa3' = pound symbol in utf-8 - # u'\xa3' = pound symbol in latin-1 (iso-8859-1) + # '\xa3' = pound symbol in unicode + # '\xc2\xa3' = pound symbol in utf-8 + # '\xa3' = pound symbol in latin-1 (iso-8859-1) - meta = u'' - head = u'' + meta + u'' - body_content = u'\xa3' - body = u'' + body_content + u'' - html = u'' + head + body + u'' + meta = '' + head = '' + meta + '' + body_content = '\xa3' + body = '' + body_content + '' + html = '' + head + body + '' encoding = 'utf-8' html_utf8 = html.encode(encoding) headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3']) + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence @@ -88,8 +88,8 @@ class SelectorTestCase(unittest.TestCase): """Check that classes are using slots and are weak-referenceable""" x = Selector(text='') weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % x.__class__.__name__ + assert not hasattr(x, '__dict__'), f"{x.__class__.__name__} does not use __slots__" def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, 'received both response and text'): - Selector(TextResponse(url='http://example.com', body=b''), text=u'') + Selector(TextResponse(url='http://example.com', body=b''), text='') diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 6e56a28f5..916fe012a 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -385,6 +385,38 @@ class SettingsTest(unittest.TestCase): self.assertIn('key', mydict) self.assertEqual(mydict['key'], 'val') + def test_passing_objects_as_values(self): + from scrapy.core.downloader.handlers.file import FileDownloadHandler + from scrapy.utils.misc import create_instance + from scrapy.utils.test import get_crawler + + class TestPipeline(): + def process_item(self, i, s): + return i + + settings = Settings({ + 'ITEM_PIPELINES': { + TestPipeline: 800, + }, + 'DOWNLOAD_HANDLERS': { + 'ftp': FileDownloadHandler, + }, + }) + + self.assertIn('ITEM_PIPELINES', settings.attributes) + + mypipeline, priority = settings.getdict('ITEM_PIPELINES').popitem() + self.assertEqual(priority, 800) + self.assertEqual(mypipeline, TestPipeline) + self.assertIsInstance(mypipeline(), TestPipeline) + self.assertEqual(mypipeline().process_item('item', None), 'item') + + myhandler = settings.getdict('DOWNLOAD_HANDLERS').pop('ftp') + self.assertEqual(myhandler, FileDownloadHandler) + myhandler_instance = create_instance(myhandler, None, get_crawler()) + self.assertIsInstance(myhandler_instance, FileDownloadHandler) + self.assertTrue(hasattr(myhandler_instance, 'download_request')) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_signals.py b/tests/test_signals.py index d6ae526be..a43f00b27 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -13,7 +13,7 @@ class ItemSpider(Spider): def start_requests(self): for index in range(10): - yield Request(self.mockserver.url('/status?n=200&id=%d' % index), + yield Request(self.mockserver.url(f'/status?n=200&id={index}'), meta={'index': index}) def parse(self, response): diff --git a/tests/test_spider.py b/tests/test_spider.py index e880dbd52..a76e4f76b 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -152,13 +152,13 @@ class XMLFeedSpiderTest(SpiderTest): output = list(spider._parse(response)) self.assertEqual(len(output), 2, iterator) self.assertEqual(output, [ - {'loc': [u'http://www.example.com/Special-Offers.html'], - 'updated': [u'2009-08-16'], - 'custom': [u'fuu'], - 'other': [u'bar']}, + {'loc': ['http://www.example.com/Special-Offers.html'], + 'updated': ['2009-08-16'], + 'custom': ['fuu'], + 'other': ['bar']}, {'loc': [], - 'updated': [u'2009-08-16'], - 'other': [u'foo'], + 'updated': ['2009-08-16'], + 'other': ['foo'], 'custom': []}, ], iterator) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e032b247c..e449cd706 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider): bypass_status_codes = set() def __init__(self, *args, **kwargs): - super(_HttpErrorSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [ self.mockserver.url("/status?n=200"), self.mockserver.url("/status?n=404"), diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 79eda35b3..2f454addc 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -163,11 +163,11 @@ class GeneratorOutputChainSpider(Spider): class _GeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -175,12 +175,12 @@ class _GeneratorDoNothingMiddleware: class GeneratorFailMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r raise LookupError() def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} @@ -192,11 +192,11 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): class GeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} @@ -229,12 +229,12 @@ class _NotGeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -243,13 +243,13 @@ class NotGeneratorFailMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) raise ReferenceError() return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] @@ -262,12 +262,12 @@ class NotGeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 5141f47af..9456b01d4 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -385,7 +385,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): class TestSettingsCustomPolicy(TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + settings = {'REFERRER_POLICY': CustomPythonOrgPolicy} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index e5d3ef582..061bc8c7c 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -50,8 +50,9 @@ class BuildComponentListTest(unittest.TestCase): def test_duplicate_components_in_list(self): duplicate_list = ['a', 'b', 'a'] - self.assertRaises(ValueError, build_component_list, None, - duplicate_list, convert=lambda x: x) + with self.assertRaises(ValueError) as cm: + build_component_list(None, duplicate_list, convert=lambda x: x) + self.assertIn(str(duplicate_list), str(cm.exception)) def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence @@ -141,6 +142,22 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli(settings, ['-:pickle']) ) + def test_feed_export_config_overwrite(self): + settings = Settings() + self.assertEqual( + {'output.json': {'format': 'json', 'overwrite': True}}, + feed_process_params_from_cli(settings, [], None, ['output.json']) + ) + + def test_output_and_overwrite_output(self): + with self.assertRaises(UsageError): + feed_process_params_from_cli( + Settings(), + ['output1.json'], + None, + ['output2.json'], + ) + def test_feed_complete_default_values_from_settings_empty(self): feed = {} settings = Settings({ @@ -149,6 +166,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_URI_PARAMS": (1, 2, 3, 4), + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -157,6 +175,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "indent": 42, "store_empty": True, "uri_params": (1, 2, 3, 4), + "batch_item_count": 2, }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -169,6 +188,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -177,6 +197,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "indent": 42, "store_empty": True, "uri_params": None, + "batch_item_count": 2, }) diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 6b05c8771..f5d684d3f 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -16,7 +16,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): try: Request(**result) except TypeError as e: - self.fail("Request kwargs are not correct {}".format(e)) + self.fail(f"Request kwargs are not correct {e}") def test_get(self): curl_command = "curl http://example.org/" diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index aa18ef1f3..e4bccf30e 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -299,7 +299,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache = LocalWeakReferencedCache() refs = [] for x in range(max): - refs.append(Request('https://example.org/{}'.format(x))) + refs.append(Request(f'https://example.org/{x}')) cache[refs[-1]] = x self.assertEqual(len(cache), max) for i, r in enumerate(refs): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 8c84331b9..e60242a3b 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -40,15 +40,15 @@ class MustbeDeferredTest(unittest.TestCase): def cb1(value, arg1, arg2): - return "(cb1 %s %s %s)" % (value, arg1, arg2) + return f"(cb1 {value} {arg1} {arg2})" def cb2(value, arg1, arg2): - return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2)) + return defer.succeed(f"(cb2 {value} {arg1} {arg2})") def cb3(value, arg1, arg2): - return "(cb3 %s %s %s)" % (value, arg1, arg2) + return f"(cb3 {value} {arg1} {arg2})" def cb_fail(value, arg1, arg2): @@ -56,7 +56,7 @@ def cb_fail(value, arg1, arg2): def eb1(failure, arg1, arg2): - return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2) + return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" class DeferUtilsTest(unittest.TestCase): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index d17bb2cbc..79f5a2bbe 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -7,9 +7,6 @@ from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = "foo{}bar".format(os.linesep) - - class XmliterTestCase(unittest.TestCase): xmliter = staticmethod(xmliter) @@ -54,7 +51,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 - body = u""" + body = """ <þingflokkar> <þingflokkur id="26"> @@ -97,15 +94,15 @@ class XmliterTestCase(unittest.TestCase): XmlResponse(url="http://example.com", body=body, encoding='utf-8'), ): attrs = [] - for x in self.xmliter(r, u'þingflokkur'): + for x in self.xmliter(r, 'þingflokkur'): attrs.append((x.attrib['id'], - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(), - x.xpath(u'./tímabil/fyrstaþing/text()').getall())) + x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(), + x.xpath('./tímabil/fyrstaþing/text()').getall())) self.assertEqual(attrs, - [(u'26', [u'-'], [u'80']), - (u'21', [u'Ab'], [u'76']), - (u'27', [u'A'], [u'27'])]) + [('26', ['-'], ['80']), + ('21', ['Ab'], ['76']), + ('27', ['A'], ['27'])]) def test_xmliter_text(self): body = ( @@ -114,7 +111,7 @@ class XmliterTestCase(unittest.TestCase): ) self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], - [[u'one'], [u'two']]) + [['one'], ['two']]) def test_xmliter_namespaces(self): body = b""" @@ -179,7 +176,7 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse('http://www.example.com', body=body) self.assertEqual( next(self.xmliter(response, 'item')).get(), - u'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' + 'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' ) @@ -265,10 +262,10 @@ class UtilsCsvTestCase(unittest.TestCase): result = [row for row in csv] self.assertEqual(result, - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, + {'id': '4', 'name': 'empty', 'value': ''}]) # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: @@ -281,10 +278,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter='\t') self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_quotechar(self): body1 = get_testdata('feeds', 'feed-sample6.csv') @@ -294,19 +291,19 @@ class UtilsCsvTestCase(unittest.TestCase): csv1 = csviter(response1, quotechar="'") self.assertEqual([row for row in csv1], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, + {'id': '4', 'name': 'empty', 'value': ''}]) response2 = TextResponse(url="http://example.com/", body=body2) csv2 = csviter(response2, delimiter="|", quotechar="'") self.assertEqual([row for row in csv2], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_wrong_quotechar(self): body = get_testdata('feeds', 'feed-sample6.csv') @@ -314,10 +311,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual([row for row in csv], - [{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"}, - {u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"}, - {u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"}, - {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}]) + [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, + {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"}, + {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, + {"'id'": "4", "'name'": "'empty'", "'value'": ""}]) def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') @@ -325,10 +322,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter='\t') self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_headers(self): sample = get_testdata('feeds', 'feed-sample3.csv').splitlines() @@ -338,10 +335,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, headers=[h.decode('utf-8') for h in headers]) self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': 'foo\nbar'}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_falserow(self): body = get_testdata('feeds', 'feed-sample3.csv') @@ -351,10 +348,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_exception(self): body = get_testdata('feeds', 'feed-sample3.csv') @@ -377,8 +374,8 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual( list(csv), [ - {u'id': u'1', u'name': u'latin1', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}, + {'id': '1', 'name': 'latin1', 'value': 'test'}, + {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'}, ] ) @@ -387,8 +384,8 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual( list(csv), [ - {u'id': u'1', u'name': u'cp852', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, + {'id': '1', 'name': 'cp852', 'value': 'test'}, + {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, ] ) @@ -412,7 +409,7 @@ class TestHelper(unittest.TestCase): def _assert_type_and_value(self, a, b, obj): self.assertTrue(type(a) is type(b), - 'Got {}, expected {} for {!r}'.format(type(a), type(b), obj)) + f'Got {type(a)}, expected {type(b)} for { obj!r}') self.assertEqual(a, b) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 9bb996d27..e95a3a316 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -12,11 +12,22 @@ __doctests__ = ['scrapy.utils.misc'] class UtilsMiscTestCase(unittest.TestCase): - def test_load_object(self): + def test_load_object_class(self): + obj = load_object(Field) + self.assertIs(obj, Field) + obj = load_object('scrapy.item.Field') + self.assertIs(obj, Field) + + def test_load_object_function(self): + obj = load_object(load_object) + self.assertIs(obj, load_object) obj = load_object('scrapy.utils.misc.load_object') - assert obj is load_object + self.assertIs(obj, load_object) + + def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') + self.assertRaises(TypeError, load_object, dict()) def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index bdbec1beb..2be38620c 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -29,9 +29,28 @@ class UtilsMiscPy3TestCase(unittest.TestCase): yield 1 yield from g() + def m(): + yield 1 + + def helper(): + return 0 + + yield helper() + + def n(): + yield 1 + + def helper(): + return 0 + + yield helper() + return 2 + assert is_generator_with_return_value(f) assert is_generator_with_return_value(g) assert not is_generator_with_return_value(h) assert not is_generator_with_return_value(i) assert not is_generator_with_return_value(j) assert not is_generator_with_return_value(k) # not recursive + assert not is_generator_with_return_value(m) + assert is_generator_with_return_value(n) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ebce3c079..3115cc92f 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -3,8 +3,8 @@ import gc import operator import platform import unittest +from datetime import datetime from itertools import count -from sys import version_info from warnings import catch_warnings from scrapy.utils.python import ( @@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase): class ToUnicodeTest(unittest.TestCase): def test_converting_an_utf8_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e') + self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e') def test_converting_a_latin_1_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e') + self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e') def test_converting_a_unicode_to_unicode_should_return_the_same_object(self): - self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e') + self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e') def test_converting_a_strange_object_should_raise_TypeError(self): self.assertRaises(TypeError, to_unicode, 423) @@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase): def test_errors_argument(self): self.assertEqual( to_unicode(b'a\xedb', 'utf-8', errors='replace'), - u'a\ufffdb' + 'a\ufffdb' ) class ToBytesTest(unittest.TestCase): def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self): - self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49') + self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49') def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): - self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49') + self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49') def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self): self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e') @@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase): def test_errors_argument(self): self.assertEqual( - to_bytes(u'a\ufffdb', 'latin-1', errors='replace'), + to_bytes('a\ufffdb', 'latin-1', errors='replace'), b'a?b' ) @@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase): assert binary_is_text(b"hello") def test_utf_16_strings_contain_null_bytes(self): - assert binary_is_text(u"hello".encode('utf-16')) + assert binary_is_text("hello".encode('utf-16')) def test_one_with_encoding(self): assert binary_is_text(b"
    Price \xa3
    ") @@ -179,6 +179,9 @@ class UtilsPythonTestCase(unittest.TestCase): def f2(a, b=None, c=None): pass + def f3(a, b=None, *, c=None): + pass + class A: def __init__(self, a, b, c): pass @@ -199,6 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(f1), ['a', 'b', 'c']) self.assertEqual(get_func_args(f2), ['a', 'b', 'c']) + self.assertEqual(get_func_args(f3), ['a', 'b', 'c']) self.assertEqual(get_func_args(A), ['a', 'b', 'c']) self.assertEqual(get_func_args(a.method), ['a', 'b', 'c']) self.assertEqual(get_func_args(partial_f1), ['b', 'c']) @@ -212,15 +216,15 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(str.split), []) self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) - else: - self.assertEqual( - get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) - self.assertEqual( - get_func_args(operator.itemgetter(2), stripself=True), ['obj']) - if version_info < (3, 6): - self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) - else: + elif platform.python_implementation() == 'PyPy': + self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj']) + + build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y') + if build_date >= datetime(2020, 4, 7): # PyPy 3.6-v7.3.1 self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) + else: + self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 450e4bdca..ee68cf6b1 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase): method="POST", body=b"some body", headers={'content-encoding': 'text/html; charset=latin-1'}, - cookies={'currency': u'руб'}, + cookies={'currency': 'руб'}, encoding='latin-1', priority=20, meta={'a': 'b'}, @@ -102,6 +102,12 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_delegated_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider.delegated_callback, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) @@ -132,6 +138,11 @@ class TestSpiderMixin: pass +class TestSpiderDelegation: + def delegated_callback(self, response): + pass + + def parse_item(response): pass @@ -155,6 +166,9 @@ class TestSpider(Spider, TestSpiderMixin): __parse_item_reference = private_parse_item __handle_error_reference = private_handle_error + def __init__(self): + self.delegated_callback = TestSpiderDelegation().delegated_callback + def parse_item(self, response): pass diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5a52dd695..5ff2e41ef 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): def test_simple_render(self): context = dict(project_name='proj', name='spi', classname='TheSpider') - template = u'from ${project_name}.spiders.${name} import ${classname}' - rendered = u'from proj.spiders.spi import TheSpider' + template = 'from ${project_name}.spiders.${name} import ${classname}' + rendered = 'from proj.spiders.spi import TheSpider' template_path = os.path.join(self.tmp_path, 'templ.py.tmpl') render_path = os.path.join(self.tmp_path, 'templ.py') diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 2f885a0e8..144c7bd76 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -213,7 +213,7 @@ def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) assert url.startswith(args[1]), \ - 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (args[0], url, args[1]) + f'Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`' return do_expected @@ -254,7 +254,7 @@ for k, args in enumerate( start=1, ): t_method = create_guess_scheme_t(args) - t_method.__name__ = 'test_uri_%03d' % k + t_method.__name__ = f'test_uri_{k:03}' setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation @@ -269,7 +269,7 @@ for k, args in enumerate( start=1, ): t_method = create_skipped_scheme_t(args) - t_method.__name__ = 'test_uri_skipped_%03d' % k + t_method.__name__ = f'test_uri_skipped_{k:03}' setattr(GuessSchemeTest, t_method.__name__, t_method) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index ee64d455c..a60181a3a 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -253,7 +253,7 @@ class WebClientTestCase(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) + return f"http://127.0.0.1:{self.portno}/{path}" def testPayload(self): s = "0123456789" * 10 @@ -265,7 +265,7 @@ class WebClientTestCase(unittest.TestCase): # it should extract from url return defer.gatherResults([ getPage(self.getURL("host")).addCallback( - self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)), + self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")), getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( self.assertEqual, to_bytes("www.example.com"))]) @@ -298,7 +298,7 @@ class WebClientTestCase(unittest.TestCase): """ d = getPage(self.getURL("host"), timeout=100) d.addCallback( - self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)) + self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")) return d def test_timeoutTriggering(self): @@ -376,7 +376,7 @@ class WebClientSSLTestCase(unittest.TestCase): interface="127.0.0.1") def getURL(self, path): - return "https://127.0.0.1:%d/%s" % (self.portno, path) + return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): self.tmpname = self.mktemp() diff --git a/tox.ini b/tox.ini index 4557c63e3..12e40295c 100644 --- a/tox.ini +++ b/tox.ini @@ -12,8 +12,9 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras - botocore>=1.3.23 - Pillow>=3.4.2 + boto3>=1.13.0 + botocore>=1.4.87 + Pillow>=4.0.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -75,9 +76,9 @@ deps = zope.interface==4.1.3 -rtests/requirements-py3.txt # Extras - botocore==1.3.23 + botocore==1.4.87 google-cloud-storage==1.29.0 - Pillow==3.4.2 + Pillow==4.0.0 [testenv:pinned] deps = @@ -88,7 +89,7 @@ deps = basepython = python3 deps = {[pinned]deps} - # First lxml version that includes a Windows wheel for Python 3.5, so we do + # First lxml version that includes a Windows wheel for Python 3.6, so we do # not need to build lxml from sources in a CI Windows job: lxml==3.8.0