diff --git a/.bandit.yml b/.bandit.yml index 243379b0b..41f1bb597 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -8,6 +8,7 @@ skips: - B311 - B320 - B321 +- B324 - B402 # https://github.com/scrapy/scrapy/issues/4180 - B403 - B404 diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d9e4a2831..1d9b9c02f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.5.0 +current_version = 2.6.1 commit = True tag = True tag_name = {new_version} diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6bdfcb5dc..98fa44c7f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,10 +8,10 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.9 + - python-version: "3.10" env: TOXENV: security - - python-version: 3.9 + - python-version: "3.10" env: TOXENV: flake8 # Pylint requires installing reppy, which does not support Python 3.9 @@ -19,11 +19,10 @@ jobs: - python-version: 3.8 env: TOXENV: pylint - TOX_PIP_VERSION: 20.3.3 - - python-version: 3.9 + - python-version: 3.6 env: TOXENV: typing - - python-version: 3.8 # Keep in sync with .readthedocs.yml + - python-version: "3.10" # Keep in sync with .readthedocs.yml env: TOXENV: docs @@ -38,8 +37,5 @@ jobs: - name: Run check env: ${{ matrix.env }} run: | - if [[ ! -z "$TOX_PIP_VERSION" ]]; then - pip install tox-pip-version - fi pip install -U tox tox diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b48066ea4..44b682830 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: "3.10" - name: Check Tag id: check-release-tag diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 095ca1013..3aaf688c7 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 81beda5da..1fc8d914b 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -17,10 +17,16 @@ jobs: - python-version: 3.9 env: TOXENV: py + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio - python-version: pypy3 env: TOXENV: pypy3 - PYPY_VERSION: 3.6-v7.3.1 + PYPY_VERSION: 3.6-v7.3.3 # pinned deps - python-version: 3.6.12 @@ -40,15 +46,6 @@ jobs: - python-version: 3.8 env: TOXENV: extra-deps - TOX_PIP_VERSION: 20.3.3 - - # 3.10-pre - - python-version: "3.10.0-beta.4" - env: - TOXENV: py - - python-version: "3.10.0-beta.4" - env: - TOXENV: asyncio steps: - uses: actions/checkout@v2 @@ -75,9 +72,6 @@ jobs: $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - if [[ ! -z "$TOX_PIP_VERSION" ]]; then - pip install tox-pip-version - fi pip install -U tox tox diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 30fda33e8..ab7385118 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -17,10 +17,15 @@ jobs: - python-version: 3.8 env: TOXENV: py - # https://twistedmatrix.com/trac/ticket/9990 - #- python-version: 3.9 - #env: - #TOXENV: py + - python-version: 3.9 + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio steps: - uses: actions/checkout@v2 diff --git a/.readthedocs.yml b/.readthedocs.yml index 80a1cd036..390be3749 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,12 +5,13 @@ sphinx: fail_on_warning: true build: - image: latest + os: ubuntu-20.04 + tools: + # For available versions, see: + # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python + python: "3.10" # Keep in sync with .github/workflows/checks.yml python: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image - version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - requirements: docs/requirements.txt - path: . diff --git a/README.rst b/README.rst index 5750e2c0f..6b563d638 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,5 @@ +.. image:: https://scrapy.org/img/scrapylogo.png + ====== Scrapy ====== diff --git a/conftest.py b/conftest.py index 05b4ccdad..117087790 100644 --- a/conftest.py +++ b/conftest.py @@ -75,6 +75,12 @@ def only_asyncio(request, reactor_pytest): pytest.skip('This test is only run with --reactor=asyncio') +@pytest.fixture(autouse=True) +def only_not_asyncio(request, reactor_pytest): + if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio': + pytest.skip('This test is only run without --reactor=asyncio') + + def pytest_configure(config): if config.getoption("--reactor") == "asyncio": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/conf.py b/docs/conf.py index 406c4d94a..55aa72d5a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [ r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions # Never documented before, and deprecated now. - r'^scrapy\.item\.DictItem$', r'^scrapy\.linkextractors\.FilteringLinkExtractor$', # Implementation detail of LxmlLinkExtractor @@ -304,10 +303,14 @@ intersphinx_mapping = { hoverxref_auto_ref = True hoverxref_role_types = { "class": "tooltip", + "command": "tooltip", "confval": "tooltip", "hoverxref": "tooltip", "mod": "tooltip", "ref": "tooltip", + "reqmeta": "tooltip", + "setting": "tooltip", + "signal": "tooltip", } hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal'] diff --git a/docs/conftest.py b/docs/conftest.py index 8c735e838..a0636f8ac 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE from scrapy.http.response.html import HtmlResponse from sybil import Sybil -from sybil.parsers.codeblock import CodeBlockParser +try: + # >2.0.1 + from sybil.parsers.codeblock import PythonCodeBlockParser +except ImportError: + from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser from sybil.parsers.doctest import DocTestParser from sybil.parsers.skip import skip @@ -21,7 +25,7 @@ def setup(namespace): pytest_collect_file = Sybil( parsers=[ DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE), - CodeBlockParser(future_imports=['print_function']), + PythonCodeBlockParser(future_imports=['print_function']), skip, ], pattern='*.rst', diff --git a/docs/faq.rst b/docs/faq.rst index 8283cab11..8a9ba809b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -94,15 +94,6 @@ How can I scrape an item with attributes in different pages? See :ref:`topics-request-response-ref-request-callback-arguments`. - -Scrapy crashes with: ImportError: No module named win32api ----------------------------------------------------------- - -You need to install `pywin32`_ because of `this Twisted bug`_. - -.. _pywin32: https://sourceforge.net/projects/pywin32/ -.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707 - How can I simulate a user login in my spider? --------------------------------------------- diff --git a/docs/index.rst b/docs/index.rst index 7647b3781..75e08f537 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,8 @@ testing. .. _web crawling: https://en.wikipedia.org/wiki/Web_crawler .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping +.. _getting-help: + Getting help ============ @@ -24,12 +26,14 @@ Having trouble? We'd like to help! * Search for questions on the archives of the `scrapy-users mailing list`_. * Ask a question in the `#scrapy IRC channel`_, * Report bugs with Scrapy in our `issue tracker`_. +* Join the Discord community `Scrapy Discord`_. .. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users .. _Scrapy subreddit: https://www.reddit.com/r/scrapy/ .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues +.. _Scrapy Discord: https://discord.gg/mv3yErfpvq First steps @@ -155,7 +159,6 @@ Solving specific problems topics/debug topics/contracts topics/practices - topics/avoiding-bans topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -180,9 +183,6 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. -:doc:`topics/avoiding-bans` - Avoid getting banned from websites. - :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. diff --git a/docs/intro/examples.rst b/docs/intro/examples.rst index 96363c7d5..edff894c6 100644 --- a/docs/intro/examples.rst +++ b/docs/intro/examples.rst @@ -7,7 +7,7 @@ Examples The best way to learn is with examples, and Scrapy is no exception. For this reason, there is an example Scrapy project named quotesbot_, that you can use to play and learn more about Scrapy. It contains two spiders for -http://quotes.toscrape.com, one using CSS selectors and another one using XPath +https://quotes.toscrape.com, one using CSS selectors and another one using XPath expressions. The quotesbot_ project is available at: https://github.com/scrapy/quotesbot. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 8581dde0b..b8d3a16bc 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -190,7 +190,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to successfully install Scrapy and its dependencies. Here are some proposed solutions: -* *(Recommended)* **Don't** use system python, install a new, updated version +* *(Recommended)* **Don't** use system Python. Install a new, updated version that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: @@ -234,8 +234,8 @@ For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. This means that these dependencies will be built during installation. -On macOS, you are likely to face an issue with building Cryptography dependency, -solution to this problem is described +On macOS, you are likely to face an issue with building the Cryptography +dependency. The solution to this problem is described `here `_, that is to ``brew install openssl`` and then export the flags that this command recommends (only needed when installing Scrapy). Installing on Linux has no special diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index dd80c7bd0..f3d652621 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -4,7 +4,7 @@ Scrapy at a glance ================== -Scrapy is an application framework for crawling web sites and extracting +Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting structured data which can be used for a wide range of useful applications, like data mining, information processing or historical archival. @@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. Here's the code for a spider that scrapes famous quotes from website -http://quotes.toscrape.com, following the pagination:: +https://quotes.toscrape.com, following the pagination:: import scrapy @@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination:: class QuotesSpider(scrapy.Spider): name = 'quotes' start_urls = [ - 'http://quotes.toscrape.com/tag/humor/', + 'https://quotes.toscrape.com/tag/humor/', ] def parse(self, response): diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 438f3d6df..cde1b1ef4 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -7,7 +7,7 @@ Scrapy Tutorial In this tutorial, we'll assume that Scrapy is already installed on your system. If that's not the case, see :ref:`intro-install`. -We are going to scrape `quotes.toscrape.com `_, a website +We are going to scrape `quotes.toscrape.com `_, a website that lists quotes from famous authors. This tutorial will walk you through these tasks: @@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named def start_requests(self): urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -143,9 +143,9 @@ similar to this:: 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened 2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) 2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023 - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished) @@ -184,8 +184,8 @@ for your spider:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -207,7 +207,7 @@ Extracting data The best way to learn how to extract data with Scrapy is trying selectors using the :ref:`Scrapy shell `. Run:: - scrapy shell 'http://quotes.toscrape.com/page/1/' + scrapy shell 'https://quotes.toscrape.com/page/1/' .. note:: @@ -217,18 +217,18 @@ using the :ref:`Scrapy shell `. Run:: On Windows, use double quotes instead:: - scrapy shell "http://quotes.toscrape.com/page/1/" + scrapy shell "https://quotes.toscrape.com/page/1/" You will see something like:: [ ... Scrapy log here ... ] - 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler [s] item {} - [s] request - [s] response <200 http://quotes.toscrape.com/page/1/> + [s] request + [s] response <200 https://quotes.toscrape.com/page/1/> [s] settings [s] spider [s] Useful shortcuts: @@ -241,7 +241,7 @@ object: .. invisible-code-block: python - response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html') + response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html') >>> response.css('title') [] @@ -277,9 +277,19 @@ As an alternative, you could've written: >>> response.css('title::text')[0].get() 'Quotes to Scrape' -However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList` -instance avoids an ``IndexError`` and returns ``None`` when it doesn't -find any element matching the selection. +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +raise an :exc:`IndexError` exception if there are no results:: + + >>> response.css('noelement')[0].get() + Traceback (most recent call last): + ... + IndexError: list index out of range + +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +if there are no results:: + +>>> response.css("noelement").get() There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail @@ -345,7 +355,7 @@ Extracting quotes and authors Now that you know a bit about selection and extraction, let's complete our spider by writing the code to extract the quotes from the web page. -Each quote in http://quotes.toscrape.com is represented by HTML elements that look +Each quote in https://quotes.toscrape.com is represented by HTML elements that look like this: .. code-block:: html @@ -369,7 +379,7 @@ like this: Let's open up scrapy shell and play a bit to find out how to extract the data we want:: - $ scrapy shell 'http://quotes.toscrape.com' + $ scrapy shell 'https://quotes.toscrape.com' We get a list of selectors for the quote HTML elements with: @@ -434,8 +444,8 @@ in the callback, as you can see below:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -448,9 +458,9 @@ in the callback, as you can see below:: If you run this spider, it will output the extracted data with the log:: - 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} - 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"} @@ -478,7 +488,7 @@ The `JSON Lines`_ format is useful because it's stream-like, you can easily append new records to it. It doesn't have the same problem of JSON when you run twice. Also, as each record is a separate line, you can process big files without having to fit everything in memory, there are tools like `JQ`_ to help -doing that at the command-line. +do that at the command-line. In small projects (like the one in this tutorial), that should be enough. However, if you want to perform more complex things with the scraped items, you @@ -495,7 +505,7 @@ Following links =============== Let's say, instead of just scraping the stuff from the first two pages -from http://quotes.toscrape.com, you want quotes from all the pages in the website. +from https://quotes.toscrape.com, you want quotes from all the pages in the website. Now that you know how to extract data from pages, let's see how to follow links from them. @@ -539,7 +549,7 @@ page, extracting data from it:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/1/', ] def parse(self, response): @@ -590,7 +600,7 @@ As a shortcut for creating Request objects you can use class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/1/', ] def parse(self, response): @@ -644,7 +654,7 @@ this time for scraping author information:: class AuthorSpider(scrapy.Spider): name = 'author' - start_urls = ['http://quotes.toscrape.com/'] + start_urls = ['https://quotes.toscrape.com/'] def parse(self, response): author_page_links = response.css('.author + a') @@ -717,7 +727,7 @@ with a specific tag, building the URL based on the argument:: name = "quotes" def start_requests(self): - url = 'http://quotes.toscrape.com/' + url = 'https://quotes.toscrape.com/' tag = getattr(self, 'tag', None) if tag is not None: url = url + 'tag/' + tag @@ -737,7 +747,7 @@ with a specific tag, building the URL based on the argument:: If you pass the ``tag=humor`` argument to this spider, you'll notice that it will only visit URLs from the ``humor`` tag, such as -``http://quotes.toscrape.com/tag/humor``. +``https://quotes.toscrape.com/tag/humor``. You can :ref:`learn more about handling spider arguments here `. diff --git a/docs/news.rst b/docs/news.rst index 0ea412e75..5d92067b5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,450 @@ Release notes ============= +.. _release-2.6.1: + +Scrapy 2.6.1 (2022-03-01) +------------------------- + +Fixes a regression introduced in 2.6.0 that would unset the request method when +following redirects. + + +.. _release-2.6.0: + +Scrapy 2.6.0 (2022-03-01) +------------------------- + +Highlights: + +* :ref:`Security fixes for cookie handling <2.6-security-fixes>` + +* Python 3.10 support + +* :ref:`asyncio support ` is no longer considered + experimental, and works out-of-the-box on Windows regardless of your Python + version + +* Feed exports now support :class:`pathlib.Path` output paths and per-feed + :ref:`item filtering ` and + :ref:`post-processing ` + +.. _2.6-security-fixes: + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- When a :class:`~scrapy.http.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.http.Request` object to be + scheduled, the cookies defined in the original + :class:`~scrapy.http.Request` object are no longer copied into the new + :class:`~scrapy.http.Request` object. + + If you manually set the ``Cookie`` header on a + :class:`~scrapy.http.Request` object and the domain name of the redirect + URL is not an exact match for the domain of the URL of the original + :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.http.Request` object. + + The old behavior could be exploited by an attacker to gain access to your + cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more + information. + + .. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8 + + .. note:: It is still possible to enable the sharing of cookies between + different domains with a shared domain suffix (e.g. + ``example.com`` and any subdomain) by defining the shared domain + suffix (e.g. ``example.com``) as the cookie domain when defining + your cookies. See the documentation of the + :class:`~scrapy.http.Request` class for more information. + +- When the domain of a cookie, either received in the ``Set-Cookie`` header + of a response or defined in a :class:`~scrapy.http.Request` object, is set + to a `public suffix `_, the cookie is now + ignored unless the cookie domain is the same as the request domain. + + The old behavior could be exploited by an attacker to inject cookies from a + controlled domain into your cookiejar that could be sent to other domains + not controlled by the attacker. Please, see the `mfjm-vh54-3f96 security + advisory`_ for more information. + + .. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96 + + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The h2_ dependency is now optional, only needed to + :ref:`enable HTTP/2 support `. (:issue:`5113`) + + .. _h2: https://pypi.org/project/h2/ + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The ``formdata`` parameter of :class:`~scrapy.FormRequest`, if specified + for a non-POST request, now overrides the URL query string, instead of + being appended to it. (:issue:`2919`, :issue:`3579`) + +- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, now + the return value of that function, and not the ``params`` input parameter, + will determine the feed URI parameters, unless that return value is + ``None``. (:issue:`4962`, :issue:`4966`) + +- In :class:`scrapy.core.engine.ExecutionEngine`, methods + :meth:`~scrapy.core.engine.ExecutionEngine.crawl`, + :meth:`~scrapy.core.engine.ExecutionEngine.download`, + :meth:`~scrapy.core.engine.ExecutionEngine.schedule`, + and :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle` + now raise :exc:`RuntimeError` if called before + :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`. (:issue:`5090`) + + These methods used to assume that + :attr:`ExecutionEngine.slot ` had + been defined by a prior call to + :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`, so they were + raising :exc:`AttributeError` instead. + +- If the API of the configured :ref:`scheduler ` does not + meet expectations, :exc:`TypeError` is now raised at startup time. Before, + other exceptions would be raised at run time. (:issue:`3559`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- ``scrapy.http.TextResponse.body_as_unicode``, deprecated in Scrapy 2.2, has + now been removed. (:issue:`5393`) + +- ``scrapy.item.BaseItem``, deprecated in Scrapy 2.2, has now been removed. + (:issue:`5398`) + +- ``scrapy.item.DictItem``, deprecated in Scrapy 1.8, has now been removed. + (:issue:`5398`) + +- ``scrapy.Spider.make_requests_from_url``, deprecated in Scrapy 1.4, has now + been removed. (:issue:`4178`, :issue:`4356`) + + +Deprecations +~~~~~~~~~~~~ + +- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, + returning ``None`` or modifying the ``params`` input parameter is now + deprecated. Return a new dictionary instead. (:issue:`4962`, :issue:`4966`) + +- :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`) + + - Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new + :meth:`Request.to_dict ` method. + + - Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new + :func:`scrapy.utils.request.request_from_dict` function. + +- In :mod:`scrapy.squeues`, the following queue classes are deprecated: + :class:`~scrapy.squeues.PickleFifoDiskQueueNonRequest`, + :class:`~scrapy.squeues.PickleLifoDiskQueueNonRequest`, + :class:`~scrapy.squeues.MarshalFifoDiskQueueNonRequest`, + and :class:`~scrapy.squeues.MarshalLifoDiskQueueNonRequest`. You should + instead use: + :class:`~scrapy.squeues.PickleFifoDiskQueue`, + :class:`~scrapy.squeues.PickleLifoDiskQueue`, + :class:`~scrapy.squeues.MarshalFifoDiskQueue`, + and :class:`~scrapy.squeues.MarshalLifoDiskQueue`. (:issue:`5117`) + +- Many aspects of :class:`scrapy.core.engine.ExecutionEngine` that come from + a time when this class could handle multiple :class:`~scrapy.Spider` + objects at a time have been deprecated. (:issue:`5090`) + + - The :meth:`~scrapy.core.engine.ExecutionEngine.has_capacity` method + is deprecated. + + - The :meth:`~scrapy.core.engine.ExecutionEngine.schedule` method is + deprecated, use :meth:`~scrapy.core.engine.ExecutionEngine.crawl` or + :meth:`~scrapy.core.engine.ExecutionEngine.download` instead. + + - The :attr:`~scrapy.core.engine.ExecutionEngine.open_spiders` attribute + is deprecated, use :attr:`~scrapy.core.engine.ExecutionEngine.spider` + instead. + + - The ``spider`` parameter is deprecated for the following methods: + + - :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle` + + - :meth:`~scrapy.core.engine.ExecutionEngine.crawl` + + - :meth:`~scrapy.core.engine.ExecutionEngine.download` + + Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider` + first to set the :class:`~scrapy.Spider` object. + + +New features +~~~~~~~~~~~~ + +- You can now use :ref:`item filtering ` to control which items + are exported to each output feed. (:issue:`4575`, :issue:`5178`, + :issue:`5161`, :issue:`5203`) + +- You can now apply :ref:`post-processing ` to feeds, and + :ref:`built-in post-processing plugins ` are provided for + output file compression. (:issue:`2174`, :issue:`5168`, :issue:`5190`) + +- The :setting:`FEEDS` setting now supports :class:`pathlib.Path` objects as + keys. (:issue:`5383`, :issue:`5384`) + +- Enabling :ref:`asyncio ` while using Windows and Python 3.8 + or later will automatically switch the asyncio event loop to one that + allows Scrapy to work. See :ref:`asyncio-windows`. (:issue:`4976`, + :issue:`5315`) + +- The :command:`genspider` command now supports a start URL instead of a + domain name. (:issue:`4439`) + +- :mod:`scrapy.utils.defer` gained 2 new functions, + :func:`~scrapy.utils.defer.deferred_to_future` and + :func:`~scrapy.utils.defer.maybe_deferred_to_future`, to help :ref:`await + on Deferreds when using the asyncio reactor `. + (:issue:`5288`) + +- :ref:`Amazon S3 feed export storage ` gained + support for `temporary security credentials`_ + (:setting:`AWS_SESSION_TOKEN`) and endpoint customization + (:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`) + + .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys + +- New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file. + (:issue:`5279`) + +- :attr:`Request.cookies ` values that are + :class:`bool`, :class:`float` or :class:`int` are cast to :class:`str`. + (:issue:`5252`, :issue:`5253`) + +- You may now raise :exc:`~scrapy.exceptions.CloseSpider` from a handler of + the :signal:`spider_idle` signal to customize the reason why the spider is + stopping. (:issue:`5191`) + +- When using + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`, the + proxy URL for non-HTTPS HTTP/1.1 requests no longer needs to include a URL + scheme. (:issue:`4505`, :issue:`4649`) + +- All built-in queues now expose a ``peek`` method that returns the next + queue object (like ``pop``) but does not remove the returned object from + the queue. (:issue:`5112`) + + If the underlying queue does not support peeking (e.g. because you are not + using ``queuelib`` 1.6.1 or later), the ``peek`` method raises + :exc:`NotImplementedError`. + +- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have + an ``attributes`` attribute that makes subclassing easier. For + :class:`~scrapy.http.Request`, it also allows subclasses to work with + :func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`, + :issue:`5130`, :issue:`5218`) + +- The :meth:`~scrapy.core.scheduler.BaseScheduler.open` and + :meth:`~scrapy.core.scheduler.BaseScheduler.close` methods of the + :ref:`scheduler ` are now optional. (:issue:`3559`) + +- HTTP/1.1 :exc:`~scrapy.core.downloader.handlers.http11.TunnelError` + exceptions now only truncate response bodies longer than 1000 characters, + instead of those longer than 32 characters, making it easier to debug such + errors. (:issue:`4881`, :issue:`5007`) + +- :class:`~scrapy.loader.ItemLoader` now supports non-text responses. + (:issue:`5145`, :issue:`5269`) + + +Bug fixes +~~~~~~~~~ + +- The :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` settings + are no longer ignored if defined in :attr:`~scrapy.Spider.custom_settings`. + (:issue:`4485`, :issue:`5352`) + +- Removed a module-level Twisted reactor import that could prevent + :ref:`using the asyncio reactor `. (:issue:`5357`) + +- The :command:`startproject` command works with existing folders again. + (:issue:`4665`, :issue:`4676`) + +- The :setting:`FEED_URI_PARAMS` setting now behaves as documented. + (:issue:`4962`, :issue:`4966`) + +- :attr:`Request.cb_kwargs ` once again allows the + ``callback`` keyword. (:issue:`5237`, :issue:`5251`, :issue:`5264`) + +- Made :func:`scrapy.utils.response.open_in_browser` support more complex + HTML. (:issue:`5319`, :issue:`5320`) + +- Fixed :attr:`CSVFeedSpider.quotechar + ` being interpreted as the CSV file + encoding. (:issue:`5391`, :issue:`5394`) + +- Added missing setuptools_ to the list of dependencies. (:issue:`5122`) + + .. _setuptools: https://pypi.org/project/setuptools/ + +- :class:`LinkExtractor ` + now also works as expected with links that have comma-separated ``rel`` + attribute values including ``nofollow``. (:issue:`5225`) + +- Fixed a :exc:`TypeError` that could be raised during :ref:`feed export + ` parameter parsing. (:issue:`5359`) + + +Documentation +~~~~~~~~~~~~~ + +- :ref:`asyncio support ` is no longer considered + experimental. (:issue:`5332`) + +- Included :ref:`Windows-specific help for asyncio usage `. + (:issue:`4976`, :issue:`5315`) + +- Rewrote :ref:`topics-headless-browsing` with up-to-date best practices. + (:issue:`4484`, :issue:`4613`) + +- Documented :ref:`local file naming in media pipelines + `. (:issue:`5069`, :issue:`5152`) + +- :ref:`faq` now covers spider file name collision issues. (:issue:`2680`, + :issue:`3669`) + +- Provided better context and instructions to disable the + :setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`) + +- Documented that :ref:`reppy-parser` does not support Python 3.9+. + (:issue:`5226`, :issue:`5231`) + +- Documented :ref:`the scheduler component `. + (:issue:`3537`, :issue:`3559`) + +- Documented the method used by :ref:`media pipelines + ` to :ref:`determine if a file has expired + `. (:issue:`5120`, :issue:`5254`) + +- :ref:`run-multiple-spiders` now features + :func:`scrapy.utils.project.get_project_settings` usage. (:issue:`5070`) + +- :ref:`run-multiple-spiders` now covers what happens when you define + different per-spider values for some settings that cannot differ at run + time. (:issue:`4485`, :issue:`5352`) + +- Extended the documentation of the + :class:`~scrapy.extensions.statsmailer.StatsMailer` extension. + (:issue:`5199`, :issue:`5217`) + +- Added :setting:`JOBDIR` to :ref:`topics-settings`. (:issue:`5173`, + :issue:`5224`) + +- Documented :attr:`Spider.attribute `. + (:issue:`5174`, :issue:`5244`) + +- Documented :attr:`TextResponse.urljoin `. + (:issue:`1582`) + +- Added the ``body_length`` parameter to the documented signature of the + :signal:`headers_received` signal. (:issue:`5270`) + +- Clarified :meth:`SelectorList.get ` usage + in the :ref:`tutorial `. (:issue:`5256`) + +- The documentation now features the shortest import path of classes with + multiple import paths. (:issue:`2733`, :issue:`5099`) + +- ``quotes.toscrape.com`` references now use HTTPS instead of HTTP. + (:issue:`5395`, :issue:`5396`) + +- Added a link to `our Discord server `_ + to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`) + +- The pronunciation of the project name is now :ref:`officially + ` /ˈskreɪpaɪ/. (:issue:`5280`, :issue:`5281`) + +- Added the Scrapy logo to the README. (:issue:`5255`, :issue:`5258`) + +- Fixed issues and implemented minor improvements. (:issue:`3155`, + :issue:`4335`, :issue:`5074`, :issue:`5098`, :issue:`5134`, :issue:`5180`, + :issue:`5194`, :issue:`5239`, :issue:`5266`, :issue:`5271`, :issue:`5273`, + :issue:`5274`, :issue:`5276`, :issue:`5347`, :issue:`5356`, :issue:`5414`, + :issue:`5415`, :issue:`5416`, :issue:`5419`, :issue:`5420`) + + +Quality Assurance +~~~~~~~~~~~~~~~~~ + +- Added support for Python 3.10. (:issue:`5212`, :issue:`5221`, + :issue:`5265`) + +- Significantly reduced memory usage by + :func:`scrapy.utils.response.response_httprepr`, used by the + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` downloader + middleware, which is enabled by default. (:issue:`4964`, :issue:`4972`) + +- Removed uses of the deprecated :mod:`optparse` module. (:issue:`5366`, + :issue:`5374`) + +- Extended typing hints. (:issue:`5077`, :issue:`5090`, :issue:`5100`, + :issue:`5108`, :issue:`5171`, :issue:`5215`, :issue:`5334`) + +- Improved tests, fixed CI issues, removed unused code. (:issue:`5094`, + :issue:`5157`, :issue:`5162`, :issue:`5198`, :issue:`5207`, :issue:`5208`, + :issue:`5229`, :issue:`5298`, :issue:`5299`, :issue:`5310`, :issue:`5316`, + :issue:`5333`, :issue:`5388`, :issue:`5389`, :issue:`5400`, :issue:`5401`, + :issue:`5404`, :issue:`5405`, :issue:`5407`, :issue:`5410`, :issue:`5412`, + :issue:`5425`, :issue:`5427`) + +- Implemented improvements for contributors. (:issue:`5080`, :issue:`5082`, + :issue:`5177`, :issue:`5200`) + +- Implemented cleanups. (:issue:`5095`, :issue:`5106`, :issue:`5209`, + :issue:`5228`, :issue:`5235`, :issue:`5245`, :issue:`5246`, :issue:`5292`, + :issue:`5314`, :issue:`5322`) + + +.. _release-2.5.1: + +Scrapy 2.5.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + .. _release-2.5.0: Scrapy 2.5.0 (2021-04-06) @@ -940,9 +1384,8 @@ Bug fixes * zope.interface 5.0.0 and later versions are now supported (:issue:`4447`, :issue:`4448`) -* :meth:`Spider.make_requests_from_url - `, deprecated in Scrapy - 1.4.0, now issues a warning when used (:issue:`4412`) +* ``Spider.make_requests_from_url``, deprecated in Scrapy 1.4.0, now issues a + warning when used (:issue:`4412`) Documentation @@ -1454,6 +1897,88 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.2: + +Scrapy 1.8.2 (2022-03-01) +------------------------- + +**Security bug fixes:** + +- When a :class:`~scrapy.http.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.http.Request` object to be + scheduled, the cookies defined in the original + :class:`~scrapy.http.Request` object are no longer copied into the new + :class:`~scrapy.http.Request` object. + + If you manually set the ``Cookie`` header on a + :class:`~scrapy.http.Request` object and the domain name of the redirect + URL is not an exact match for the domain of the URL of the original + :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.http.Request` object. + + The old behavior could be exploited by an attacker to gain access to your + cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more + information. + + .. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8 + + .. note:: It is still possible to enable the sharing of cookies between + different domains with a shared domain suffix (e.g. + ``example.com`` and any subdomain) by defining the shared domain + suffix (e.g. ``example.com``) as the cookie domain when defining + your cookies. See the documentation of the + :class:`~scrapy.http.Request` class for more information. + +- When the domain of a cookie, either received in the ``Set-Cookie`` header + of a response or defined in a :class:`~scrapy.http.Request` object, is set + to a `public suffix `_, the cookie is now + ignored unless the cookie domain is the same as the request domain. + + The old behavior could be exploited by an attacker to inject cookies into + your requests to some other domains. Please, see the `mfjm-vh54-3f96 + security advisory`_ for more information. + + .. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96 + + +.. _release-1.8.1: + +Scrapy 1.8.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + .. _release-1.8.0: Scrapy 1.8.0 (2019-10-28) @@ -1754,7 +2279,7 @@ New features * A new scheduler priority queue, ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant - scheduling improvement on crawls targetting multiple web domains, at the + scheduling improvement on crawls targeting multiple web domains, at the cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) * A new :attr:`Request.cb_kwargs ` attribute @@ -2792,7 +3317,7 @@ Bug fixes - Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse ` (:issue:`2225`). - Fix for invalid JSON and XML files when spider yields no items (:issue:`872`). -- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). +- Implement ``flush()`` for ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). Refactoring ~~~~~~~~~~~ @@ -3655,7 +4180,7 @@ Scrapy 0.24.3 (2014-08-09) - adding some xpath tips to selectors docs (:commit:`2d103e0`) - fix tests to account for https://github.com/scrapy/w3lib/pull/23 (:commit:`f8d366a`) - get_func_args maximum recursion fix #728 (:commit:`81344ea`) -- Updated input/ouput processor example according to #560. (:commit:`f7c4ea8`) +- Updated input/output processor example according to #560. (:commit:`f7c4ea8`) - Fixed Python syntax in tutorial. (:commit:`db59ed9`) - Add test case for tunneling proxy (:commit:`f090260`) - Bugfix for leaking Proxy-Authorization header to remote host when using tunneling (:commit:`d8793af`) @@ -4282,7 +4807,7 @@ Code rearranged and removed - Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot - Removed support for default field values in Scrapy items (:rev:`2616`) - Removed experimental crawlspider v2 (:rev:`2632`) -- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`) +- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`) - Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`) - Removed deprecated Execution Queue (:rev:`2704`) - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) @@ -4317,7 +4842,7 @@ Scrapyd changes ~~~~~~~~~~~~~~~ - Scrapyd now uses one process per spider -- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default) +- It stores one log file per spider run, and rotate them keeping the latest 5 logs per spider (by default) - A minimal web ui was added, available at http://localhost:6800 by default - There is now a ``scrapy server`` command to start a Scrapyd server of the current project @@ -4353,7 +4878,7 @@ New features and improvements - Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195) - Support for overriding default request headers per spider (#181) - Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186) -- Splitted Debian package into two packages - the library and the service (#187) +- Split Debian package into two packages - the library and the service (#187) - Scrapy log refactoring (#188) - New extension for keeping persistent spider contexts among different runs (#203) - Added ``dont_redirect`` request.meta key for avoiding redirects (#233) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 71d027c86..0c3a7ed88 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -67,7 +67,7 @@ this: the :ref:`Scheduler ` and asks for possible next Requests to crawl. -9. The process repeats (from step 1) until there are no more requests from the +9. The process repeats (from step 3) until there are no more requests from the :ref:`Scheduler `. Components diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 82c5f271f..3a6941a2c 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -10,10 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet - recommended for production environments. Future Scrapy versions - may introduce related changes without a deprecation period or - warning. .. _install-asyncio: @@ -30,6 +26,7 @@ reactor manually. You can do that using install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + .. _using-custom-loops: Using custom asyncio loops @@ -40,4 +37,62 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the use it instead of the default asyncio event loop. +.. _asyncio-windows: +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations: + +- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required + when using Twisted. + +- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work + with Twisted. + +So on Python 3.8+ the event loop class needs to be changed. + +.. versionchanged:: 2.6.0 + The event loop class is changed automatically when you change the + :setting:`TWISTED_REACTOR` setting or call + :func:`~scrapy.utils.reactor.install_reactor`. + +To change the event loop class manually, call the following code before +installing the reactor:: + + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +You can put this in the same function that installs the reactor, if you do that +yourself, or in some code that runs before the reactor is installed, e.g. +``settings.py``. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + + +.. _asyncio-await-dfd: + +Awaiting on Deferreds +===================== + +When the asyncio reactor isn't installed, you can await on Deferreds in the +coroutines directly. When it is installed, this is not possible anymore, due to +specifics of the Scrapy coroutine integration (the coroutines are wrapped into +:class:`asyncio.Future` objects, not into +:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into +Futures. Scrapy provides two helpers for this: + +.. autofunction:: scrapy.utils.defer.deferred_to_future +.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future +.. tip:: If you need to use these functions in code that aims to be compatible + with lower versions of Scrapy that do not provide these functions, + down to Scrapy 2.0 (earlier versions do not support + :mod:`asyncio`), you can copy the implementation of these functions + into your own code. diff --git a/docs/topics/avoiding-bans.rst b/docs/topics/avoiding-bans.rst deleted file mode 100644 index 59f0da191..000000000 --- a/docs/topics/avoiding-bans.rst +++ /dev/null @@ -1,340 +0,0 @@ -.. _bans: - -============= -Avoiding bans -============= - -This topic covers some of the strategies that you can follow to avoid getting -different or bad responses from a website that you are crawling due to filters -such as regional filters, web browser filters, etc. - -.. _avoiding-crawls: - -Avoiding crawls -=============== - -The best way not to be banned from a website is not to send requests to it in -the first place. - -One way to avoid crawling a website is to find the desired dataset through -other means. For example, you can use Google’s `dataset search engine`_. - -If the target website is the only or best source of the desired information, -and you only need to extract the data on a monthly basis or a lower frequency, -you may be able to crawl a public snapshot of the target website instead. -`Common Crawl`_ is an open repository of web crawl data that you can access -freely. It contains monthly snapshots of a wide variety of websites and, if you -are lucky, your target website will be among them. - -.. _Common Crawl: https://commoncrawl.org/ -.. _dataset search engine: https://datasetsearch.research.google.com/ - - -.. _being-polite: - -Being polite -============ - -To avoid being banned, you should first avoid giving a website reasons to ban -you. - -.. _identifying-yourself: - -Identifying yourself --------------------- - -If your crawling has a noticeable negative impact on a website or you crawl -content that should not be crawled, website administrators will need to do -something. - -Set :setting:`USER_AGENT` to a value that uniquely identifies your spider and -includes contact information, so that website administrators can contact you. - - -.. _following-robotstxt: - -Following robots.txt guidelines -------------------------------- - -Some websites provide a ``robots.txt`` file at their root path (e.g. -``http://example.com/robots.txt``) that describes the guidelines that they wish -bots to follow when crawling their website. - -Before you start writing a spider for a website, read their ``robots.txt`` -file and implement your spider following its guidelines. See the `robots.txt -standard draft`_ or the `robots.txt Google specification`_ for information on -how to read ``robots.txt`` files. - -To ensure that your spider does not crawl pages restricted by ``robots.txt`` -guidelines, set :setting:`ROBOTSTXT_OBEY` to ``True`` to enable the -:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` -middleware. When you do, if your spider attempts to crawl a restricted page, -this middleware aborts that request with the following message:: - - Forbidden by robots.txt - -Also set :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and -:setting:`DOWNLOAD_DELAY` to values that comply with the ``Crawl-Delay`` or -``Request-Rate`` directives from the ``robots.txt`` guidelines. - -You may also use the :ref:`AutoThrottle extension ` on top -of that, so that when the target website experiences a high load, your spider -automatically switches to higher download delays. - -.. _robots.txt Google specification: https://developers.google.com/search/reference/robots_txt -.. _robots.txt standard draft: https://tools.ietf.org/html/draft-koster-rep-00 - - -.. _choosing-crawl-speed: - -Finding the right guidelines on your own ----------------------------------------- - -If a website does not specify a desired download delay, or does not provide a -``robots.txt`` file, you should make an effort to find out the right values for -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY` that -will not have a noticeable negative impact on the target website. - -Use a service like `SimilarWeb`_ to find out the amount of monthly traffic that -the target website receives, and choose concurrency and delay values that will -not cause a noticeable traffic increase. - -.. _SimilarWeb: https://www.similarweb.com - - -.. _filters-and-challenges: - -Bypassing filters and solving challenges -======================================== - -Some websites implement filters and challenges that aim to deny access or alter -their content based on aspects of the visitor, such as the country where they -are or the web browsing tool they use. - -.. _regional-filter: - -Bypassing regional filters --------------------------- - -Some websites send different or bad responses based on the region or country -associated to your `IP address`_. - -To bypass these filters, get access to a `proxy server`_ that has an outgoing -IP address from a region that gets the desired responses. - -Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` -middleware to configure your spider to use that proxy. - -.. _IP address: https://en.wikipedia.org/wiki/IP_address -.. _proxy server: https://en.wikipedia.org/wiki/Proxy_server - - -.. _web-browser-filter: - -Bypassing web browser filters ------------------------------ - -Some websites send different or bad responses if they detect that your request -does not come from a web browser. - -To bypass these filters, switch your :setting:`USER_AGENT` to a value copied -from those that popular web browsers use. In some rare cases, you may need a -user agent string from a specific web browser. - -There are multiple Scrapy plugins that can rotate your requests through popular -web browser user agent strings, such as scrapy-fake-useragent_, -scrapy-random-useragent_ or Scrapy-UserAgents_. - -For advanced web browser filters, -:ref:`pre-rendering JavaScript ` or -:ref:`using a headless browser ` may be necessary. -Use these options only as a last resort, however, because they cause a higher -load per request on the target website. - -.. _scrapy-fake-useragent: https://github.com/alecxe/scrapy-fake-useragent -.. _scrapy-random-useragent: https://github.com/cleocn/scrapy-random-useragent -.. _Scrapy-UserAgents: https://pypi.org/project/Scrapy-UserAgents/ - - -.. _request-delay-filter: - -Bypassing request delay filters -------------------------------- - -Some websites may ban your IP after they detect that your requests use a -constant download delay. - -To help bypassing these filters, the :setting:`RANDOMIZE_DOWNLOAD_DELAY` -setting is enabled by default. When that is not enough, an -:ref:`IP address rotation solution ` may be much more effective. - - -.. _isp-filter: - -Bypassing internet service provider filters -------------------------------------------- - -Some websites send different or bad responses if they detect that your request -comes from an IP address that belongs to a `data center`_, as opposed to a -residential IP address from an `internet service provider`_ or a mobile IP -address from a `mobile network`_. - -To bypass these filters, get access to a proxy server that has an outgoing IP -address that is either residential or mobile. Note that you may also get -different responses depending on whether your IP address is residential or -mobile. - -Use the :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` -middleware to configure your spider to use that proxy. - -.. _data center: https://en.wikipedia.org/wiki/Data_center -.. _internet service provider: https://en.wikipedia.org/wiki/Internet_service_provider -.. _mobile network: https://en.wikipedia.org/wiki/Cellular_network - - -.. _captcha: - -Solving CAPTCHA challenges --------------------------- - -Some websites require you to solve a `CAPTCHA challenge`_ to get the desired -response. - -To bypass these filters, several options exist: - -- You could have your spider present the CAPTCHA challenge to you and wait - for you to solve it manually. - -- Some CAPTCHA challenges can be solved using an `optical character - recognition`_ (OCR) solution such as pytesseract_. - -- Paid CAPTCHA solving services exist. - -Whichever solution you choose, implement it as a :ref:`downloader middleware -` that automatically detects CAPTCHA challenges -in responses and solves them, so that your spider code only receives successful -responses. - -.. _CAPTCHA challenge: https://en.wikipedia.org/wiki/CAPTCHA -.. _optical character recognition: https://en.wikipedia.org/wiki/Optical_character_recognition -.. _pytesseract: https://github.com/madmaze/pytesseract - - -.. _ip-rotation: - -IP address rotation solutions -============================= - -See below some of the different solutions there are to have your requests use -different outgoing IP addresses. - -When using this approach, remember to set :setting:`COOKIES_ENABLED` to -``False`` to disable global cookie handling. This prevents websites from -identifying two requests as coming from the same user agent even if they come -from different IP addresses and have different user-agent strings. You can -still include some cookies manually in your requests. Define them through the -``Cookies`` header of your requests. See -:class:`Request.headers `. - -.. _smart-proxy: - -Smart proxies -------------- - -An increasing number of websites use solutions that apply many of the above -filters and challenges at the same time. - -There are paid proxy services, like `Zyte Smart Proxy Manager`_, that -automatically bypass website filters and challenges, so that your spider only -gets successful responses. They also allow managing sessions to simulate user -behavior. - -For Zyte Smart Proxy Manager, installing scrapy-crawlera_ will offer advanced -integration with Scrapy. For other services, use the -:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` middleware -or implement your own :ref:`downloader middleware -`. - -.. _scrapy-crawlera: https://scrapy-crawlera.readthedocs.io/en/latest/ -.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ - - -.. _rotating-proxy: - -Rotating proxies ----------------- - -Rotating proxy services like ProxyMesh_ send different requests through -different proxies. This can decrease the likelihood of being affected by some -filters or challenges. - -.. _ProxyMesh: https://proxymesh.com/ - - -.. _free-proxies: - -Free proxies ------------- - -You can easily find lists of free proxies in the internet, and you can use -a solution like `scrapy-rotating-proxies`_ to configure multiple proxies in -your spider and have requests rotate through them automatically. - -This approach, however, has serious drawbacks: - -- Free proxies may stop working at any moment. You need to implement a way to - refresh your list of free proxies. - -- In addition to handling occasional bad responses from websites, you - need to handle all kinds of bad responses from proxies. You may even need - to inspect the response body to determine if a response comes from the - target website or from a misbehaving proxy. - -- Advanced antibot solutions may automatically detect and filter out traffic - from free proxies. - -.. _scrapy-rotating-proxies: https://github.com/TeamHG-Memex/scrapy-rotating-proxies - - -.. _custom-rotating-proxy: - -Custom rotating proxy ---------------------- - -If you have spare servers, you can set them up as proxies and use scrapoxy_ to -build a custom proxy that rotates traffic through them. However, the initial -setup can be complex, and your requests will be vulnerable to -:ref:`internet service provider filtering `. - -.. _scrapoxy: https://scrapoxy.io/ - - -.. _tor: - -The Tor network ---------------- - -It is possible to send requests through the `Tor network`_. - -The initial setup to have Scrapy working with Tor is not straightforward. -Use a search engine to find up-to-date documentation specific to using -Scrapy and Tor together. - -The main drawback of using the Tor network is that traffic can be extremely -slow. - -.. _Tor network: https://en.wikipedia.org/wiki/Tor_(anonymity_network) - - -.. _commercial-support: - -Seeking professional help -========================= - -Avoiding bans, filters and challenges can be difficult and tricky, and may -sometimes require special infrastructure. - -If you find yourself unable to prevent your spider from getting bad responses, -consider contacting `commercial support`_. - -.. _commercial support: https://scrapy.org/support/ diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index eef6b36ff..8c0b8e55f 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -230,10 +230,16 @@ Usage example:: genspider --------- -* Syntax: ``scrapy genspider [-t template] `` +* Syntax: ``scrapy genspider [-t template] `` * Requires project: *no* -Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. +.. versionadded:: 2.6.0 + The ability to pass a URL instead of a domain. + +Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. + +.. note:: Even if an HTTPS URL is specified, the protocol used in + ``start_urls`` is always HTTP. This is a known issue: :issue:`3553`. Usage example:: diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 0904637b0..2aef755c7 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -75,23 +75,28 @@ coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. This means you can use many useful Python libraries providing such code:: - class MySpider(Spider): + class MySpiderDeferred(Spider): # ... - async def parse_with_deferred(self, response): + async def parse(self, response): additional_response = await treq.get('https://additional.url') additional_data = await treq.content(additional_response) # ... use response and additional_data to yield items and requests - async def parse_with_asyncio(self, response): + class MySpiderAsyncio(Spider): + # ... + async def parse(self, response): async with aiohttp.ClientSession() as session: async with session.get('https://additional.url') as additional_response: - additional_data = await r.text() + additional_data = await additional_response.text() # ... use response and additional_data to yield items and requests .. note:: Many libraries that use coroutines, such as `aio-libs`_, require the :mod:`asyncio` loop and to use them you need to :doc:`enable asyncio support in Scrapy`. +.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor, + you need to :ref:`wrap them`. + Common use cases for asynchronous code include: * requesting data from websites, databases and other services (in callbacks, diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 057b1ec62..9bf97c628 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -19,14 +19,14 @@ Caveats with inspecting the live browser DOM Since Developer Tools operate on a live browser DOM, what you'll actually see when inspecting the page source is not the original HTML, but a modified one -after applying some browser clean up and executing Javascript code. Firefox, +after applying some browser clean up and executing JavaScript code. Firefox, in particular, is known for adding ```` elements to tables. Scrapy, on the other hand, does not modify the original page HTML, so you won't be able to extract any data if you use ```` in your XPath expressions. Therefore, you should keep in mind the following things: -* Disable Javascript while inspecting the DOM looking for XPaths to be +* Disable JavaScript while inspecting the DOM looking for XPaths to be used in Scrapy (in the Developer Tools settings click `Disable JavaScript`) * Never use full XPath paths, use relative and clever ones based on attributes @@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class= "text"`` we will see the quote-text we clicked on. The `Inspector` lets you copy XPaths to selected elements. Let's try it out. -First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal: +First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal: .. code-block:: none - $ scrapy shell "http://quotes.toscrape.com/" + $ scrapy shell "https://quotes.toscrape.com/" Then, back to your web browser, right-click on the ``span`` tag, select ``Copy > XPath`` and paste it in the Scrapy shell like so: .. invisible-code-block: python - response = load_response('http://quotes.toscrape.com/', 'quotes.html') + response = load_response('https://quotes.toscrape.com/', 'quotes.html') >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] @@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the type ``json``. If we click on this request, we see that the request URL is -``http://quotes.toscrape.com/api/quotes?page=1`` and the response +``https://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click on the request and open ``Open in new tab`` to get a better overview. @@ -247,7 +247,7 @@ also request each page to get every quote on the site:: name = 'quote' allowed_domains = ['quotes.toscrape.com'] page = 1 - start_urls = ['http://quotes.toscrape.com/api/quotes?page=1'] + start_urls = ['https://quotes.toscrape.com/api/quotes?page=1'] def parse(self, response): data = json.loads(response.text) @@ -255,7 +255,7 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" + url = f"https://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) This spider starts at the first page of the quotes-API. With each @@ -280,7 +280,7 @@ request:: from scrapy import Request request = Request.from_curl( - "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" + "curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce" "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X" "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM" @@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools -.. _quotes.toscrape.com: http://quotes.toscrape.com -.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll -.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10 +.. _quotes.toscrape.com: https://quotes.toscrape.com +.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll +.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 99d57bda9..29e350651 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -89,7 +89,7 @@ object gives you access, for example, to the :ref:`settings `. methods of installed middleware is always called on every response. If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling - process_request methods and reschedule the returned request. Once the newly returned + :meth:`process_request` methods and reschedule the returned request. Once the newly returned request is performed, the appropriate middleware chain will be called on the downloaded response. @@ -323,8 +323,21 @@ HttpAuthMiddleware This middleware authenticates all requests generated from certain spiders using `Basic access authentication`_ (aka. HTTP auth). - To enable HTTP authentication from certain spiders, set the ``http_user`` - and ``http_pass`` attributes of those spiders. + To enable HTTP authentication for a spider, set the ``http_user`` and + ``http_pass`` spider attributes to the authentication data and the + ``http_auth_domain`` spider attribute to the domain which requires this + authentication (its subdomains will be also handled in the same way). + You can set ``http_auth_domain`` to ``None`` to enable the + authentication for all requests but you risk leaking your authentication + credentials to unrelated domains. + + .. warning:: + In previous Scrapy versions HttpAuthMiddleware sent the authentication + data with all requests, which is a security problem if the spider + makes requests to several different domains. Currently if the + ``http_auth_domain`` attribute is not set, the middleware will use the + domain of the first request, which will work for some spiders but not + for others. In the future the middleware will produce an error instead. Example:: @@ -334,6 +347,7 @@ HttpAuthMiddleware http_user = 'someuser' http_pass = 'somepass' + http_auth_domain = 'intranet.example.com' name = 'intranet.example.com' # .. rest of the spider code omitted ... @@ -352,7 +366,7 @@ HttpCacheMiddleware This middleware provides low-level cache to all HTTP requests and responses. It has to be combined with a cache storage backend as well as a cache policy. - Scrapy ships with three HTTP cache storage backends: + Scrapy ships with the following HTTP cache storage backends: * :ref:`httpcache-storage-fs` * :ref:`httpcache-storage-dbm` @@ -690,14 +704,15 @@ HttpCompressionMiddleware sent/received from web sites. This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is + `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt -.. _brotlipy: https://pypi.org/project/brotlipy/ +.. _brotli: https://pypi.org/project/Brotli/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt .. _zstandard: https://pypi.org/project/zstandard/ + HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1005,7 +1020,7 @@ Parsers vary in several aspects: (shorter) rule Performance comparison of different parsers is available at `the following link -`_. +`_. .. _protego-parser: diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 8c30122b6..923336769 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -122,7 +122,7 @@ Example:: class ProductXmlExporter(XmlItemExporter): def serialize_field(self, field, name, value): - if field == 'price': + if name == 'price': return f'$ {str(value)}' return super().serialize_field(field, name, value) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2b3217d62..9a13eb82f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -272,12 +272,13 @@ in multiple files, with the specified maximum item count per file. That way, as soon as a file reaches the maximum item count, that file is delivered to the feed URI, allowing item delivery to start way before the end of the crawl. + .. _item-filter: Item filtering ============== -.. versionadded:: VERSION +.. versionadded:: 2.6.0 You can filter items that you want to allow for a particular feed by using the ``item_classes`` option in :ref:`feeds options `. Only items of @@ -312,6 +313,63 @@ ItemFilter :members: +.. _post-processing: + +Post-Processing +=============== + +.. versionadded:: 2.6.0 + +Scrapy provides an option to activate plugins to post-process feeds before they are exported +to feed storages. In addition to using :ref:`builtin plugins `, you +can create your own :ref:`plugins `. + +These plugins can be activated through the ``postprocessing`` option of a feed. +The option must be passed a list of post-processing plugins in the order you want +the feed to be processed. These plugins can be declared either as an import string +or with the imported class of the plugin. Parameters to plugins can be passed +through the feed options. See :ref:`feed options ` for examples. + +.. _builtin-plugins: + +Built-in Plugins +---------------- + +.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin + +.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin + +.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin + +.. _custom-plugins: + +Custom Plugins +-------------- + +Each plugin is a class that must implement the following methods: + +.. method:: __init__(self, file, feed_options) + + Initialize the plugin. + + :param file: file-like object having at least the `write`, `tell` and `close` methods implemented + + :param feed_options: feed-specific :ref:`options ` + :type feed_options: :class:`dict` + +.. method:: write(self, data) + + Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file. + It must return number of bytes written. + +.. method:: close(self) + + Close the target file object. + +To pass a parameter to your plugin, use :ref:`feed options `. You +can then access those parameters from the ``__init__`` method of your plugin. + + Settings ======== @@ -368,10 +426,12 @@ For instance:: 'encoding': 'latin1', 'indent': 8, }, - pathlib.Path('items.csv'): { + pathlib.Path('items.csv.gz'): { 'format': 'csv', 'fields': ['price', 'name'], 'item_filter': 'myproject.filters.MyCustomFilter2', + 'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 5, }, } @@ -397,13 +457,13 @@ as a fallback value if that key is not provided for a specific feed definition: If undefined or empty, all items are exported. - .. versionadded:: VERSION + .. versionadded:: 2.6.0 - ``item_filter``: a :ref:`filter class ` to filter items to export. :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default. - .. versionadded:: VERSION + .. versionadded:: 2.6.0 - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. @@ -435,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition: - ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. +- ``postprocessing``: list of :ref:`plugins ` to use for post-processing. + + The plugins will be used in the order of the list passed. + + .. versionadded:: 2.6.0 .. setting:: FEED_EXPORT_ENCODING @@ -679,6 +744,9 @@ The function signature should be as follows: :param spider: source spider of the feed items :type spider: scrapy.Spider + .. caution:: The function should return a new dictionary, modifying + the received ``params`` in-place is deprecated. + For example, to include the :attr:`name ` of the source spider in the feed URI: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 5351a2293..391751364 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -190,6 +190,8 @@ item. import scrapy from itemadapter import ItemAdapter + from scrapy.utils.defer import maybe_deferred_to_future + class ScreenshotPipeline: """Pipeline that uses Splash to render screenshot of @@ -202,7 +204,7 @@ item. encoded_item_url = quote(adapter["url"]) screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url) - response = await spider.crawler.engine.download(request, spider) + response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider)) if response.status != 200: # Error happened, return item. diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index e49f37a2f..f16d306c7 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command:: scrapy crawl somespider -s JOBDIR=crawls/somespider-1 +.. _topics-keeping-persistent-state-between-batches: + Keeping persistent state between batches ======================================== diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index c0f534493..0d63700c8 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -56,7 +56,7 @@ chapter `:: l.add_xpath('name', '//div[@class="product_name"]') l.add_xpath('name', '//div[@class="product_title"]') l.add_xpath('price', '//p[@id="price"]') - l.add_css('stock', 'p#stock]') + l.add_css('stock', 'p#stock') l.add_value('last_updated', 'today') # you can also use literal values return l.load_item() diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index dda04dc4d..3bf23d5f5 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -143,6 +143,7 @@ Logging settings These settings can be used to configure the logging: * :setting:`LOG_FILE` +* :setting:`LOG_FILE_APPEND` * :setting:`LOG_ENABLED` * :setting:`LOG_ENCODING` * :setting:`LOG_LEVEL` @@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If :setting:`LOG_FILE` is set, messages sent through the root logger will be redirected to a file named :setting:`LOG_FILE` with encoding :setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log -messages will be displayed on the standard error. Lastly, if +messages will be displayed on the standard error. If :setting:`LOG_FILE` is set +and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten +(discarding the output from previous runs, if any). Lastly, if :setting:`LOG_ENABLED` is ``False``, there won't be any visible log output. :setting:`LOG_LEVEL` determines the minimum level of severity to display, those @@ -215,7 +218,7 @@ For example, let's say you're scraping a website which returns many HTTP 404 and 500 responses, and you want to hide all messages like this:: 2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring - response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code + response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code is not handled or not allowed The first thing to note is a logger name - it is in brackets: diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 46bd2859b..7dff78390 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -356,6 +356,8 @@ setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used. Additional features =================== +.. _file-expiration: + File expiration --------------- @@ -383,6 +385,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key: and pipeline class MyPipeline will have expiration time set to 180. +The last modified time from the file is used to determine the age of the file in days, +which is then compared to the set expiration time to determine if the file is expired. + .. _topics-images-thumbnails: Thumbnail generation for images diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index a7a6fd129..7313c9246 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -180,8 +180,8 @@ Same example but running the spiders sequentially by chaining the deferreds: # Your second spider definition ... - configure_logging() settings = get_project_settings() + configure_logging(settings) runner = CrawlerRunner(settings) @defer.inlineCallbacks @@ -193,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds: crawl() reactor.run() # the script will block here until the last crawl call is finished +Different spiders can set different values for the same setting, but when they +run in the same process it may be impossible, by design or because of some +limitations, to use these different values. What happens in practice is +different for different settings: + +* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value + (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the + default one) cannot be read from the per-spider settings. These are applied + when the :class:`~scrapy.crawler.CrawlerRunner` or + :class:`~scrapy.crawler.CrawlerProcess` object is created. +* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first + available value is used, and if a spider requests a different reactor an + exception will be raised. These are applied when the reactor is installed. +* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the + ones used by the resolver (:setting:`DNSCACHE_ENABLED`, + :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy) + the first available value is used. These are applied when the reactor is + started. + .. seealso:: :ref:`run-from-script`. .. _distributed-crawls: @@ -226,5 +245,39 @@ crawl:: curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2 curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3 +.. _bans: +Avoiding getting banned +======================= + +Some websites implement certain measures to prevent bots from crawling them, +with varying degrees of sophistication. Getting around those measures can be +difficult and tricky, and may sometimes require special infrastructure. Please +consider contacting `commercial support`_ if in doubt. + +Here are some tips to keep in mind when dealing with these kinds of sites: + +* rotate your user agent from a pool of well-known ones from browsers (google + around to get a list of them) +* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use + cookies to spot bot behaviour +* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting. +* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites + directly +* use a pool of rotating IPs. For example, the free `Tor project`_ or paid + services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a + super proxy that you can attach your own proxies to. +* use a highly distributed downloader that circumvents bans internally, so you + can just focus on parsing clean pages. One example of such downloaders is + `Zyte Smart Proxy Manager`_ + +If you are still unable to prevent your bot getting banned, consider contacting +`commercial support`_. + +.. _Tor project: https://www.torproject.org/ +.. _commercial support: https://scrapy.org/support/ +.. _ProxyMesh: https://proxymesh.com/ +.. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders +.. _scrapoxy: https://scrapoxy.io/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d3e08efd4..92a471faf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -110,6 +110,10 @@ Request objects :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. + .. versionadded:: 2.6.0 + Cookie values that are :class:`bool`, :class:`float` or :class:`int` + are casted to :class:`str`. + :type cookies: dict or list :param encoding: the encoding of this request (defaults to ``'utf-8'``). @@ -300,7 +304,7 @@ errors if needed:: "http://www.httpbin.org/status/404", # Not found error "http://www.httpbin.org/status/500", # server issue "http://www.httpbin.org:12345/", # non-responding host, timeout expected - "http://www.httphttpbinbin.org/", # DNS error expected + "https://example.invalid/", # DNS error expected ] def start_requests(self): @@ -950,6 +954,14 @@ TextResponse objects Returns a Python object from deserialized JSON document. The result is cached after the first call. + .. method:: TextResponse.urljoin(url) + + Constructs an absolute url by combining the Response's base url with + a possible relative url. The base url shall be extracted from the + ```` tag, or just the Response's :attr:`url` if there is no such + tag. + + HtmlResponse objects -------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2ab2020fa..4e105642d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1026,6 +1026,16 @@ Default: ``None`` File name to use for logging output. If ``None``, standard error will be used. +.. setting:: LOG_FILE_APPEND + +LOG_FILE_APPEND +--------------- + +Default: ``True`` + +If ``False``, the log file specified with :setting:`LOG_FILE` will be +overwritten (discarding the output from previous runs, if any). + .. setting:: LOG_FORMAT LOG_FORMAT @@ -1566,7 +1576,7 @@ If a reactor is already installed, :meth:`CrawlerRunner.__init__ ` raises :exc:`Exception` if the installed reactor does not match the -:setting:`TWISTED_REACTOR` setting; therfore, having top-level +:setting:`TWISTED_REACTOR` setting; therefore, having top-level :mod:`~twisted.internet.reactor` imports in project files and imported third-party libraries will make Scrapy raise :exc:`Exception` when it checks which reactor is installed. @@ -1587,7 +1597,7 @@ In order to use the reactor installed by Scrapy:: def start_requests(self): reactor.callLater(self.timeout, self.stop) - urls = ['http://quotes.toscrape.com/page/1'] + urls = ['https://quotes.toscrape.com/page/1'] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -1615,7 +1625,7 @@ which raises :exc:`Exception`, becomes:: from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) - urls = ['http://quotes.toscrape.com/page/1'] + urls = ['https://quotes.toscrape.com/page/1'] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -1628,10 +1638,9 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will not attempt to install any specific reactor, and the -default reactor defined by Twisted for the current platform will be used. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +means that Scrapy will install the default reactor defined by Twisted for the +current platform. This is to maintain backward compatibility and avoid possible +problems caused by using a non-default reactor. For additional information, see :doc:`core/howto/choosing-reactor`. @@ -1645,8 +1654,19 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` -The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 +The maximum URL length to allow for crawled URLs. + +This setting can act as a stopping condition in case of URLs of ever-increasing +length, which may be caused for example by a programming error either in the +target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and +:setting:`DEPTH_LIMIT`. + +Use ``0`` to allow URLs of any length. + +The default value is copied from the `Microsoft Internet Explorer maximum URL +length`_, even though this setting exists for different reasons. + +.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT @@ -1658,7 +1678,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"`` The default User-Agent to use when crawling, unless overridden. This user agent is also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and -there is no overridding User-Agent header specified for the request. +there is no overriding User-Agent header specified for the request. Settings documented elsewhere: diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 8c90a506c..007e9fc2f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -99,7 +99,7 @@ Available Shortcuts shortcuts - ``fetch(url[, redirect=True])`` - fetch a new response from the given URL - and update all related objects accordingly. You can optionaly ask for HTTP + and update all related objects accordingly. You can optionally ask for HTTP 3xx redirections to not be followed by passing ``redirect=False`` - ``fetch(request)`` - fetch a new response from the given request and update diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index a67cc1879..2fbd0b51c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -60,7 +60,7 @@ Let's take an example:: class SignalSpider(scrapy.Spider): name = 'signals' - start_urls = ['http://quotes.toscrape.com/page/1/'] + start_urls = ['https://quotes.toscrape.com/page/1/'] @classmethod def from_crawler(cls, crawler, *args, **kwargs): @@ -413,7 +413,7 @@ headers_received .. versionadded:: 2.5 .. signal:: headers_received -.. function:: headers_received(headers, request, spider) +.. function:: headers_received(headers, body_length, request, spider) Sent by the HTTP 1.1 and S3 download handlers when the response headers are available for a given request, before downloading any additional content. diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f0158dc41..f27bc79c0 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,8 +122,8 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.Request` objects and :ref:`item object - `. + iterable of :class:`~scrapy.Request` or :ref:`item ` + objects. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_spider_exception` in the following @@ -440,4 +440,3 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. - diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 67b9e2e0e..ece02ae47 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -42,15 +42,12 @@ Even though this cycle applies (more or less) to any kind of spider, there are different kinds of default spiders bundled into Scrapy for different purposes. We will talk about those types here. -.. module:: scrapy.spiders - :synopsis: Spiders base class, spider manager and spider middleware - .. _topics-spiders-ref: scrapy.Spider ============= -.. class:: scrapy.spiders.Spider() +.. class:: scrapy.spiders.Spider .. class:: scrapy.Spider() This is the simplest spider, and the one from which every other spider @@ -122,6 +119,11 @@ scrapy.Spider send log messages through it as described on :ref:`topics-logging-from-spiders`. + .. attribute:: state + + A dict you can use to persist some spider state between batches. + See :ref:`topics-keeping-persistent-state-between-batches` to know more about it. + .. method:: from_crawler(crawler, *args, **kwargs) This is the class method used by Scrapy to create your spiders. @@ -367,7 +369,7 @@ CrawlSpider described below. If multiple rules match the same link, the first one will be used, according to the order they're defined in this attribute. - This spider also exposes an overrideable method: + This spider also exposes an overridable method: .. method:: parse_start_url(response, **kwargs) @@ -529,7 +531,7 @@ XMLFeedSpider itertag = 'n:url' # ... - Apart from these new attributes, this spider has the following overrideable + Apart from these new attributes, this spider has the following overridable methods too: .. method:: adapt_response(response) diff --git a/docs/versioning.rst b/docs/versioning.rst index 57643ea9a..9d02757b0 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C* large changes. * *B* is the release number. This will include many changes including features and things that possibly break backward compatibility, although we strive to - keep theses cases at a minimum. + keep these cases at a minimum. * *C* is the bugfix release number. Backward-incompatibilities are explicitly mentioned in the :ref:`release notes `, diff --git a/extras/qpsclient.py b/extras/qpsclient.py index f9fb70342..28703650d 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -1,5 +1,5 @@ """ -A spider that generate light requests to meassure QPS troughput +A spider that generate light requests to meassure QPS throughput usage: diff --git a/pylintrc b/pylintrc index a44712507..2cdd6321e 100644 --- a/pylintrc +++ b/pylintrc @@ -105,12 +105,14 @@ disable=abstract-method, unnecessary-lambda, unnecessary-pass, unreachable, + unspecified-encoding, unsubscriptable-object, unused-argument, unused-import, unused-private-member, unused-variable, unused-wildcard-import, + use-implicit-booleaness-not-comparison, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, diff --git a/pytest.ini b/pytest.ini index 6de08c78d..ae2ed2029 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,5 +20,4 @@ addopts = --ignore=docs/utils markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed -filterwarnings= - ignore::DeprecationWarning:twisted.web.test.test_webclient + only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed diff --git a/scrapy/VERSION b/scrapy/VERSION index 437459cd9..6a6a3d8e3 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.5.0 +2.6.1 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 8a8065bf2..396f98219 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -29,7 +29,7 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version if sys.version_info < (3, 6): - print("Scrapy %s requires Python 3.6+" % __version__) + print(f"Scrapy {__version__} requires Python 3.6+") sys.exit(1) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 91482ce01..491c4beab 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,13 +1,13 @@ import sys import os -import optparse +import argparse import cProfile import inspect import pkg_resources import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.commands import ScrapyCommand +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings @@ -123,8 +123,6 @@ def execute(argv=None, settings=None): inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) - parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), - conflict_handler='resolve') if not cmdname: _print_commands(settings, inproject) sys.exit(0) @@ -133,12 +131,14 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser.usage = f"scrapy {cmdname} {cmd.syntax()}" - parser.description = cmd.long_desc() + parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, + usage=f"scrapy {cmdname} {cmd.syntax()}", + conflict_handler='resolve', + description=cmd.long_desc()) settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings cmd.add_options(parser) - opts, args = parser.parse_args(args=argv[1:]) + opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) cmd.crawler_process = CrawlerProcess(settings) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 6e77551c6..fb304b8c0 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -2,7 +2,7 @@ Base class for Scrapy commands """ import os -from optparse import OptionGroup +import argparse from typing import Any, Dict from twisted.python import failure @@ -43,14 +43,14 @@ class ScrapyCommand: def long_desc(self): """A long description of the command. Return short description when not - available. It cannot contain newlines, since contents will be formatted + available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() def help(self): """An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines, since no post-formatting will + "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() @@ -59,22 +59,20 @@ class ScrapyCommand: """ Populate option parse with options available for this command """ - group = OptionGroup(parser, "Global Options") - group.add_option("--logfile", metavar="FILE", - help="log file. if omitted stderr will be used") - group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help=f"log level (default: {self.settings['LOG_LEVEL']})") - group.add_option("--nolog", action="store_true", - help="disable logging completely") - group.add_option("--profile", metavar="FILE", default=None, - help="write python cProfile stats to FILE") - group.add_option("--pidfile", metavar="FILE", - help="write process ID to FILE") - group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", - help="set/override setting (may be repeated)") - group.add_option("--pdb", action="store_true", help="enable pdb on failure") - - parser.add_option_group(group) + group = parser.add_argument_group(title='Global Options') + group.add_argument("--logfile", metavar="FILE", + help="log file. if omitted stderr will be used") + group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None, + help=f"log level (default: {self.settings['LOG_LEVEL']})") + group.add_argument("--nolog", action="store_true", + help="disable logging completely") + group.add_argument("--profile", metavar="FILE", default=None, + help="write python cProfile stats to FILE") + group.add_argument("--pidfile", metavar="FILE", + help="write process ID to FILE") + group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE", + help="set/override setting (may be repeated)") + group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args, opts): try: @@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand): """ def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="append scraped items to the end of FILE (use - for stdout)") - parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append", - help="dump scraped items into FILE, overwriting any existing file") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items") + parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_argument("-o", "--output", metavar="FILE", action="append", + help="append scraped items to the end of FILE (use - for stdout)") + parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append", + help="dump scraped items into FILE, overwriting any existing file") + parser.add_argument("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand): opts.overwrite_output, ) self.settings.set('FEEDS', feeds, priority='cmdline') + + +class ScrapyHelpFormatter(argparse.HelpFormatter): + """ + Help Formatter for scrapy command line help messages. + """ + def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): + super().__init__(prog, indent_increment=indent_increment, + max_help_position=max_help_position, width=width) + + def _join_parts(self, part_strings): + parts = self.format_part_strings(part_strings) + return super()._join_parts(parts) + + def format_part_strings(self, part_strings): + """ + Underline and title case command line help message headers. + """ + if part_strings and part_strings[0].startswith("usage: "): + part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):] + headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')] + for index in headings[::-1]: + char = '-' if "Global Options" in part_strings[index] else '=' + part_strings[index] = part_strings[index][:-2].title() + underline = ''.join(["\n", (char * len(part_strings[index])), "\n"]) + part_strings.insert(index + 1, underline) + return part_strings diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index ae21d86e6..a16f4beb7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -49,10 +49,10 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-l", "--list", dest="list", action="store_true", - help="only list contracts, without checking them") - parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true', - help="print contract tests for all spiders") + parser.add_argument("-l", "--list", dest="list", action="store_true", + help="only list contracts, without checking them") + parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true', + help="print contract tests for all spiders") def run(self, args, opts): # load contracts diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 95f87e8c3..9b2ebb37f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -26,11 +26,11 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", help="use this spider") - parser.add_option("--headers", dest="headers", action="store_true", - help="print response HTTP headers instead of body") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument("--spider", dest="spider", help="use this spider") + parser.add_argument("--headers", dest="headers", action="store_true", + help="print response HTTP headers instead of body") + parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 5f44daa70..ed5f588e9 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,6 +4,7 @@ import string from importlib import import_module from os.path import join, dirname, abspath, exists, splitext +from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand @@ -22,6 +23,14 @@ def sanitize_module_name(module_name): return module_name +def extract_domain(url): + """Extract domain name from URL string""" + o = urlparse(url) + if o.scheme == '' and o.netloc == '': + o = urlparse("//" + url.lstrip("/")) + return o.netloc + + class Command(ScrapyCommand): requires_project = False @@ -35,16 +44,16 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-l", "--list", dest="list", action="store_true", - help="List available templates") - parser.add_option("-e", "--edit", dest="edit", action="store_true", - help="Edit spider after creating it") - parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE", - help="Dump template to standard output") - parser.add_option("-t", "--template", dest="template", default="basic", - help="Uses a custom template.") - parser.add_option("--force", dest="force", action="store_true", - help="If the spider already exists, overwrite it with the template") + parser.add_argument("-l", "--list", dest="list", action="store_true", + help="List available templates") + parser.add_argument("-e", "--edit", dest="edit", action="store_true", + help="Edit spider after creating it") + parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE", + help="Dump template to standard output") + parser.add_argument("-t", "--template", dest="template", default="basic", + help="Uses a custom template.") + parser.add_argument("--force", dest="force", action="store_true", + help="If the spider already exists, overwrite it with the template") def run(self, args, opts): if opts.list: @@ -59,7 +68,8 @@ class Command(ScrapyCommand): if len(args) != 2: raise UsageError() - name, domain = args[0:2] + name, url = args[0:2] + domain = extract_domain(url) module = sanitize_module_name(name) if self.settings.get('BOT_NAME') == module: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 52118db1b..a3f6b96f4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand): def add_options(self, parser): BaseRunSpiderCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", default=None, - help="use this spider without looking for one") - parser.add_option("--pipelines", action="store_true", - help="process items through pipelines") - parser.add_option("--nolinks", dest="nolinks", action="store_true", - help="don't show links to follow (extracted requests)") - parser.add_option("--noitems", dest="noitems", action="store_true", - help="don't show scraped items") - parser.add_option("--nocolour", dest="nocolour", action="store_true", - help="avoid using pygments to colorize the output") - parser.add_option("-r", "--rules", dest="rules", action="store_true", - help="use CrawlSpider rules to discover the callback") - parser.add_option("-c", "--callback", dest="callback", - help="use this callback for parsing, instead looking for a callback") - parser.add_option("-m", "--meta", dest="meta", - help="inject extra meta into the Request, it must be a valid raw json string") - parser.add_option("--cbkwargs", dest="cbkwargs", - help="inject extra callback kwargs into the Request, it must be a valid raw json string") - parser.add_option("-d", "--depth", dest="depth", type="int", default=1, - help="maximum depth for parsing requests [default: %default]") - parser.add_option("-v", "--verbose", dest="verbose", action="store_true", - help="print each depth level one by one") + parser.add_argument("--spider", dest="spider", default=None, + help="use this spider without looking for one") + parser.add_argument("--pipelines", action="store_true", + help="process items through pipelines") + parser.add_argument("--nolinks", dest="nolinks", action="store_true", + help="don't show links to follow (extracted requests)") + parser.add_argument("--noitems", dest="noitems", action="store_true", + help="don't show scraped items") + parser.add_argument("--nocolour", dest="nocolour", action="store_true", + help="avoid using pygments to colorize the output") + parser.add_argument("-r", "--rules", dest="rules", action="store_true", + help="use CrawlSpider rules to discover the callback") + parser.add_argument("-c", "--callback", dest="callback", + help="use this callback for parsing, instead looking for a callback") + parser.add_argument("-m", "--meta", dest="meta", + help="inject extra meta into the Request, it must be a valid raw json string") + parser.add_argument("--cbkwargs", dest="cbkwargs", + help="inject extra callback kwargs into the Request, it must be a valid raw json string") + parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, + help="maximum depth for parsing requests [default: %default]") + parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", + help="print each depth level one by one") @property def max_level(self): diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 8d49e440f..1b2e2601e 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -18,16 +18,16 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--get", dest="get", metavar="SETTING", - help="print raw setting value") - parser.add_option("--getbool", dest="getbool", metavar="SETTING", - help="print setting value, interpreted as a boolean") - parser.add_option("--getint", dest="getint", metavar="SETTING", - help="print setting value, interpreted as an integer") - parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", - help="print setting value, interpreted as a float") - parser.add_option("--getlist", dest="getlist", metavar="SETTING", - help="print setting value, interpreted as a list") + parser.add_argument("--get", dest="get", metavar="SETTING", + help="print raw setting value") + parser.add_argument("--getbool", dest="getbool", metavar="SETTING", + help="print setting value, interpreted as a boolean") + parser.add_argument("--getint", dest="getint", metavar="SETTING", + help="print setting value, interpreted as an integer") + parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING", + help="print setting value, interpreted as a float") + parser.add_argument("--getlist", dest="getlist", metavar="SETTING", + help="print setting value, interpreted as a list") def run(self, args, opts): settings = self.crawler_process.settings diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index d1944df3d..f67a5886a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -33,12 +33,12 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-c", dest="code", - help="evaluate the code in the shell, print the result and exit") - parser.add_option("--spider", dest="spider", - help="use this spider") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument("-c", dest="code", + help="evaluate the code in the shell, print the result and exit") + parser.add_argument("--spider", dest="spider", + help="use this spider") + parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -75,6 +75,6 @@ class Command(ScrapyCommand): def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, - kwargs={'stop_after_crawl': False}) + kwargs={'stop_after_crawl': False, 'install_signal_handlers': False}) t.daemon = True t.start() diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 1d73fa0cb..1b6374c39 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,7 +1,7 @@ import re import os import string -from importlib import import_module +from importlib.util import find_spec from os.path import join, exists, abspath from shutil import ignore_patterns, move, copy2, copystat from stat import S_IWUSR as OWNER_WRITE_PERMISSION @@ -42,11 +42,8 @@ class Command(ScrapyCommand): def _is_valid_name(self, project_name): def _module_exists(module_name): - try: - import_module(module_name) - return True - except ImportError: - return False + spec = find_spec(module_name) + return spec is not None and spec.loader is not None if not re.search(r'^[_a-zA-Z]\w*$', project_name): print('Error: Project names must begin with a letter and contain' diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 1237610cb..c6a3c273a 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -16,8 +16,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--verbose", "-v", dest="verbose", action="store_true", - help="also display twisted/python/platform info (useful for bug reports)") + parser.add_argument("--verbose", "-v", dest="verbose", action="store_true", + help="also display twisted/python/platform info (useful for bug reports)") def run(self, args, opts): if opts.verbose: diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index c8f873334..b1f52abe2 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,3 +1,4 @@ +import argparse from scrapy.commands import fetch from scrapy.utils.response import open_in_browser @@ -12,7 +13,7 @@ class Command(fetch.Command): def add_options(self, parser): super().add_options(parser) - parser.remove_option("--headers") + parser.add_argument('--headers', help=argparse.SUPPRESS) def _print_response(self, response, opts): open_in_browser(response) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 073ef16bf..b5318c7bb 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -135,11 +135,13 @@ def load_context_factory_from_settings(settings, crawler): settings=settings, crawler=crawler, ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + msg = ( + f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept " + "a `method` argument (type OpenSSL.SSL method, e.g. " + "OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` " + "argument and/or a `tls_ciphers` argument. Please, upgrade your " + "context factory class to handle them or ignore them." + ) warnings.warn(msg) return context_factory diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 8a91d4c5e..38935667d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -213,7 +213,7 @@ class TunnelingAgent(Agent): # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy - key = key + self._proxyConf + key += self._proxyConf return super()._requestWithEndpoint( key=key, endpoint=endpoint, diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index a5619d8a4..289147466 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,7 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from typing import Callable, Union +from typing import Callable, Union, cast from twisted.internet import defer from twisted.python.failure import Failure @@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @defer.inlineCallbacks def process_request(request: Request): for method in self.methods['process_request']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response for method in self.methods['process_response']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 2b8990b75..19a56d9b6 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions): verifyHostname(connection, self._hostnameASCII) except (CertificateError, VerificationError) as e: logger.warning( - 'Remote certificate is not valid for hostname "{}"; {}'.format( - self._hostnameASCII, e)) + 'Remote certificate is not valid for hostname "%s"; %s', + self._hostnameASCII, e) except ValueError as e: logger.warning( 'Ignoring error while verifying certificate ' - 'from host "{}" (exception: {})'.format( - self._hostnameASCII, repr(e))) + 'from host "%s" (exception: %r)', + self._hostnameASCII, e) DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 915cb5fe3..06cb96489 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -3,7 +3,7 @@ from time import time from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.http import HTTPClient -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.protocol import ClientFactory from scrapy.http import Headers @@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory): p.followRedirect = self.followRedirect p.afterFoundGet = self.afterFoundGet if self.timeout: + from twisted.internet import reactor timeoutCall = reactor.callLater(self.timeout, p.timeout) self.deferred.addBoth(self._cancelTimeout, timeoutCall) return p diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c2a4b702f..5c393c027 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -285,8 +285,8 @@ class Stream: self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) - bytes_to_send_size = bytes_to_send_size - chunk_size - self.metadata['remaining_content_length'] = self.metadata['remaining_content_length'] - chunk_size + bytes_to_send_size -= chunk_size + self.metadata['remaining_content_length'] -= chunk_size self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length']) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index d6d6f64f9..f40bccbb3 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -156,7 +156,7 @@ class Scraper: callback = result.request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) dfd = defer_succeed(result) - dfd.addCallback(callback, **result.request.cb_kwargs) + dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs) else: # result is a Failure result.request = request warn_on_generator_with_return_value(spider, request.errback) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 7e58521ac..7cdc28284 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Any, Callable, Generator, Iterable, Union +from typing import Any, Callable, Generator, Iterable, Union, cast from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Any: for method in self.methods['process_spider_input']: + method = cast(Callable, method) try: result = method(response=response, spider=spider) if result is not None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 578016536..a638254f1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -25,6 +25,7 @@ from scrapy.utils.log import ( configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, + log_reactor_info, log_scrapy_info, LogCounterHandler, ) @@ -38,7 +39,7 @@ logger = logging.getLogger(__name__) class Crawler: - def __init__(self, spidercls, settings=None): + def __init__(self, spidercls, settings=None, init_reactor: bool = False): if isinstance(spidercls, Spider): raise ValueError('The spidercls argument must be a class, not an object') @@ -69,6 +70,20 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + + reactor_class = self.settings.get("TWISTED_REACTOR") + if init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if reactor_class: + install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) + else: + from twisted.internet import default + default.install() + log_reactor_info() + if reactor_class: + verify_installed_reactor(reactor_class) + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() @@ -153,7 +168,6 @@ class CrawlerRunner: self._crawlers = set() self._active = set() self.bootstrap_failed = False - self._handle_twisted_reactor() @property def spiders(self): @@ -247,10 +261,6 @@ class CrawlerRunner: while self._active: yield defer.DeferredList(self._active) - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - verify_installed_reactor(self.settings["TWISTED_REACTOR"]) - class CrawlerProcess(CrawlerRunner): """ @@ -278,7 +288,6 @@ class CrawlerProcess(CrawlerRunner): def __init__(self, settings=None, install_root_handler=True): super().__init__(settings) - install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) @@ -298,7 +307,12 @@ class CrawlerProcess(CrawlerRunner): {'signame': signame}) reactor.callFromThread(self._stop_reactor) - def start(self, stop_after_crawl=True): + def _create_crawler(self, spidercls): + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings, init_reactor=True) + + def start(self, stop_after_crawl=True, install_signal_handlers=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -309,6 +323,9 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished + + :param bool install_signal_handlers: whether to install the shutdown + handlers (default: True) """ from twisted.internet import reactor if stop_after_crawl: @@ -318,6 +335,8 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) + if install_signal_handlers: + install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() @@ -337,8 +356,3 @@ class CrawlerProcess(CrawlerRunner): reactor.stop() except RuntimeError: # raised if already stopped or in shutdown stage pass - - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) - super()._handle_twisted_reactor() diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d95ed3d38..3afa06077 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,15 +1,26 @@ import logging from collections import defaultdict +from tldextract import TLDExtract + from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) +_split_domain = TLDExtract(include_psl_private_domains=True) + + +def _is_public_domain(domain): + parts = _split_domain(domain) + return not parts.domain + + class CookiesMiddleware: """This middleware enables working with sites that need cookies""" @@ -23,14 +34,29 @@ class CookiesMiddleware: raise NotConfigured return cls(crawler.settings.getbool('COOKIES_DEBUG')) + def _process_cookies(self, cookies, *, jar, request): + for cookie in cookies: + cookie_domain = cookie.domain + if cookie_domain.startswith('.'): + cookie_domain = cookie_domain[1:] + + request_domain = urlparse_cached(request).hostname.lower() + + if cookie_domain and _is_public_domain(cookie_domain): + if cookie_domain != request_domain: + continue + cookie.domain = request_domain + + jar.set_cookie_if_ok(cookie, request) + def process_request(self, request, spider): if request.meta.get('dont_merge_cookies', False): return cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] - for cookie in self._get_request_cookies(jar, request): - jar.set_cookie_if_ok(cookie, request) + cookies = self._get_request_cookies(jar, request) + self._process_cookies(cookies, jar=jar, request=request) # set Cookie header request.headers.pop('Cookie', None) @@ -44,7 +70,9 @@ class CookiesMiddleware: # extract cookies from Set-Cookie and drop invalid/expired cookies cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] - jar.extract_cookies(response, request) + cookies = jar.make_cookies(response, request) + self._process_cookies(cookies, jar=jar, request=request) + self._debug_set_cookie(response, spider) return response @@ -80,8 +108,8 @@ class CookiesMiddleware: logger.warning(msg.format(request, cookie, key)) return continue - if isinstance(cookie[key], str): - decoded[key] = cookie[key] + if isinstance(cookie[key], (bool, float, int, str)): + decoded[key] = str(cookie[key]) else: try: decoded[key] = cookie[key].decode("utf8") diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 089bf0d85..1bee3e279 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -3,10 +3,14 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +import warnings from w3lib.http import basic_auth_header from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.url import url_is_from_any_domain class HttpAuthMiddleware: @@ -24,8 +28,23 @@ class HttpAuthMiddleware: pwd = getattr(spider, 'http_pass', '') if usr or pwd: self.auth = basic_auth_header(usr, pwd) + if not hasattr(spider, 'http_auth_domain'): + warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security ' + 'problems if the spider makes requests to several different domains. http_auth_domain ' + 'will be set to the domain of the first request, please set it to the correct value ' + 'explicitly.', + category=ScrapyDeprecationWarning) + self.domain_unset = True + else: + self.domain = spider.http_auth_domain + self.domain_unset = False def process_request(self, request, spider): auth = getattr(self, 'auth', None) if auth and b'Authorization' not in request.headers: - request.headers[b'Authorization'] = auth + domain = urlparse_cached(request).hostname + if self.domain_unset: + self.domain = domain + self.domain_unset = False + if not self.domain or url_is_from_any_domain(request.url, [self.domain]): + request.headers[b'Authorization'] = auth diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 62f1c3a29..80ed7ac75 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -70,7 +70,7 @@ class HttpCacheMiddleware: self.stats.inc_value('httpcache/miss', spider=spider) if self.ignore_missing: self.stats.inc_value('httpcache/ignore', spider=spider) - raise IgnoreRequest("Ignored request not in cache: %s" % request) + raise IgnoreRequest(f"Ignored request not in cache: {request}") return None # first time request # Return cached response only if not expired diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 4053fecc5..c8c84ffb2 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string from scrapy.http import HtmlResponse +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -11,6 +12,20 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) +def _build_redirect_request(source_request, *, url, **kwargs): + redirect_request = source_request.replace( + url=url, + **kwargs, + cookies=None, + ) + if 'Cookie' in redirect_request.headers: + source_request_netloc = urlparse_cached(source_request).netloc + redirect_request_netloc = urlparse_cached(redirect_request).netloc + if source_request_netloc != redirect_request_netloc: + del redirect_request.headers['Cookie'] + return redirect_request + + class BaseRedirectMiddleware: enabled_setting = 'REDIRECT_ENABLED' @@ -47,10 +62,15 @@ class BaseRedirectMiddleware: raise IgnoreRequest("max redirections reached") def _redirect_request_using_get(self, request, redirect_url): - redirected = request.replace(url=redirect_url, method='GET', body='') - redirected.headers.pop('Content-Type', None) - redirected.headers.pop('Content-Length', None) - return redirected + redirect_request = _build_redirect_request( + request, + url=redirect_url, + method='GET', + body='', + ) + redirect_request.headers.pop('Content-Type', None) + redirect_request.headers.pop('Content-Length', None) + return redirect_request class RedirectMiddleware(BaseRedirectMiddleware): @@ -80,7 +100,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): redirected_url = urljoin(request.url, location) if response.status in (301, 307, 308) or request.method == 'HEAD': - redirected = request.replace(url=redirected_url) + redirected = _build_redirect_request(request, url=redirected_url) return self._redirect(redirected, request, spider, response.status) redirected = self._redirect_request_using_get(request, redirected_url) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index f1fdc3858..c6cc7c56d 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -2,7 +2,7 @@ An extension to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. -You can change the behaviour of this middleware by modifing the scraping settings: +You can change the behaviour of this middleware by modifying the scraping settings: RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 5479cd0e2..25fb1ed9d 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,7 +1,22 @@ from scrapy.exceptions import NotConfigured +from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr -from scrapy.utils.response import response_httprepr -from scrapy.utils.python import global_object_name + +from twisted.web import http + + +def get_header_size(headers): + size = 0 + for key, value in headers.items(): + if isinstance(value, (list, tuple)): + for v in value: + size += len(b": ") + len(key) + len(v) + return size + len(b'\r\n') * (len(headers.keys()) - 1) + + +def get_status_size(response_status): + return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15 + # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> " class DownloaderStats: @@ -24,7 +39,8 @@ class DownloaderStats: def process_response(self, request, response, spider): self.stats.inc_value('downloader/response_count', spider=spider) self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider) - reslen = len(response_httprepr(response)) + reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4 + # response.body + b"\r\n"+ response.header + b"\r\n" + response.status self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response diff --git a/scrapy/exporters.py b/scrapy/exporters.py index fb4b565cf..1c26e81db 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator from itemadapter import is_item, ItemAdapter from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import _BaseItem +from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -30,7 +30,7 @@ class BaseItemExporter: self._configure(kwargs, dont_fail=dont_fail) def _configure(self, options, dont_fail=False): - """Configure the exporter by poping options from the ``options`` dict. + """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ @@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter): return serializer(value) def _serialize_value(self, value): - if isinstance(value, _BaseItem): + if isinstance(value, Item): return self.export_item(value) elif is_item(value): return dict(self._serialize_item(value)) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 564c736f2..e7097b7a1 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,15 +11,16 @@ import sys import warnings from datetime import datetime from tempfile import NamedTemporaryFile -from typing import Any, Optional, Tuple +from typing import Any, Callable, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads from w3lib.url import file_uri_to_path from zope.interface import implementer, Interface -from scrapy import signals +from scrapy import signals, Spider from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file @@ -37,11 +38,10 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): kwargs['feed_options'] = feed_options else: warnings.warn( - "{} does not support the 'feed_options' keyword argument. Add a " + f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a " "'feed_options' parameter to its signature to remove this " "warning. This parameter will become mandatory in a future " - "version of Scrapy." - .format(builder.__qualname__), + "version of Scrapy.", category=ScrapyDeprecationWarning ) return builder(*preargs, uri, *args, **kwargs) @@ -355,32 +355,28 @@ class FeedExporter: # properly closed. return defer.maybeDeferred(slot.storage.store, slot.file) slot.finish_exporting() - logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" - log_args = {'format': slot.format, - 'itemcount': slot.itemcount, - 'uri': slot.uri} + logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" d = defer.maybeDeferred(slot.storage.store, slot.file) - # Use `largs=log_args` to copy log_args into function's scope - # instead of using `log_args` from the outer scope d.addCallback( - self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__ + self._handle_store_success, logmsg, spider, type(slot.storage).__name__ ) d.addErrback( - self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__ + self._handle_store_error, logmsg, spider, type(slot.storage).__name__ ) return d - def _handle_store_error(self, f, largs, logfmt, spider, slot_type): + def _handle_store_error(self, f, logmsg, spider, slot_type): logger.error( - logfmt % "Error storing", largs, + "Error storing %s", logmsg, exc_info=failure_to_exc_info(f), extra={'spider': spider} ) self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") - def _handle_store_success(self, f, largs, logfmt, spider, slot_type): + def _handle_store_success(self, f, logmsg, spider, slot_type): logger.info( - logfmt % "Stored", largs, extra={'spider': spider} + "Stored %s", logmsg, + extra={'spider': spider} ) self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") @@ -396,6 +392,9 @@ class FeedExporter: """ storage = self._get_storage(uri, feed_options) file = storage.open(spider) + if "postprocessing" in feed_options: + file = PostProcessingManager(feed_options["postprocessing"], file, feed_options) + exporter = self._get_exporter( file=file, format=feed_options['format'], @@ -470,10 +469,10 @@ class FeedExporter: for uri_template, values in self.feeds.items(): if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT ' + '%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT ' 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: ' - 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' - ''.format(uri_template) + 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count', + uri_template ) return False return True @@ -522,10 +521,15 @@ class FeedExporter: instance = build_instance(feedcls) method_name = '__new__' if instance is None: - raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name)) + raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") return instance - def _get_uri_params(self, spider, uri_params, slot=None): + def _get_uri_params( + self, + spider: Spider, + uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], + slot: Optional[_FeedSlot] = None, + ) -> dict: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -533,9 +537,18 @@ class FeedExporter: params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-') params['batch_time'] = utc_now.isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 - uripar_function = load_object(uri_params) if uri_params else lambda x, y: None - uripar_function(params, spider) - return params + original_params = params.copy() + uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params + new_params = uripar_function(params, spider) + if new_params is None or original_params != params: + warnings.warn( + 'Modifying the params dictionary in-place in the function defined in ' + 'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS ' + 'setting is deprecated. The function must return a new dictionary ' + 'instead.', + category=ScrapyDeprecationWarning + ) + return new_params if new_params is not None else params def _load_filter(self, feed_options): # load the item filter if declared else load the default filter class diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index e0c04b2de..d0ae29b90 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -226,7 +226,7 @@ class DbmCacheStorage: dbpath = os.path.join(self.cachedir, f'{spider.name}.db') self.db = self.dbmodule.open(dbpath, 'c') - logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider}) def close_spider(self, spider): self.db.close() @@ -280,7 +280,7 @@ class FilesystemCacheStorage: self._open = gzip.open if self.use_gzip else open def open_spider(self, spider): - logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}, + logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir}, extra={'spider': spider}) def close_spider(self, spider): diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 9de119a10..f5081a7d7 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -33,8 +33,8 @@ class MemoryUsage: self.crawler = crawler self.warned = False self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL') - self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 - self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 + self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024 + self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024 self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS') self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) @@ -77,7 +77,7 @@ class MemoryUsage: def _check_limit(self): if self.get_virtual_size() > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) - mem = self.limit/1024/1024 + mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: @@ -94,11 +94,11 @@ class MemoryUsage: self.crawler.stop() def _check_warning(self): - if self.warned: # warn only once + if self.warned: # warn only once return if self.get_virtual_size() > self.warning: self.crawler.stats.set_value('memusage/warning_reached', 1) - mem = self.warning/1024/1024 + mem = self.warning / 1024 / 1024 logger.warning("Memory usage reached %(memusage)dM", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py new file mode 100644 index 000000000..413c2e55e --- /dev/null +++ b/scrapy/extensions/postprocessing.py @@ -0,0 +1,154 @@ +""" +Extension for processing data before they are exported to feeds. +""" +from bz2 import BZ2File +from gzip import GzipFile +from io import IOBase +from lzma import LZMAFile +from typing import Any, BinaryIO, Dict, List + +from scrapy.utils.misc import load_object + + +class GzipPlugin: + """ + Compresses received data using `gzip `_. + + Accepted ``feed_options`` parameters: + + - `gzip_compresslevel` + - `gzip_mtime` + - `gzip_filename` + + See :py:class:`gzip.GzipFile` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + compress_level = self.feed_options.get("gzip_compresslevel", 9) + mtime = self.feed_options.get("gzip_mtime") + filename = self.feed_options.get("gzip_filename") + self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level, + mtime=mtime, filename=filename) + + def write(self, data: bytes) -> int: + return self.gzipfile.write(data) + + def close(self) -> None: + self.gzipfile.close() + self.file.close() + + +class Bz2Plugin: + """ + Compresses received data using `bz2 `_. + + Accepted ``feed_options`` parameters: + + - `bz2_compresslevel` + + See :py:class:`bz2.BZ2File` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + compress_level = self.feed_options.get("bz2_compresslevel", 9) + self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level) + + def write(self, data: bytes) -> int: + return self.bz2file.write(data) + + def close(self) -> None: + self.bz2file.close() + self.file.close() + + +class LZMAPlugin: + """ + Compresses received data using `lzma `_. + + Accepted ``feed_options`` parameters: + + - `lzma_format` + - `lzma_check` + - `lzma_preset` + - `lzma_filters` + + .. note:: + ``lzma_filters`` cannot be used in pypy version 7.3.1 and older. + + See :py:class:`lzma.LZMAFile` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + + format = self.feed_options.get("lzma_format") + check = self.feed_options.get("lzma_check", -1) + preset = self.feed_options.get("lzma_preset") + filters = self.feed_options.get("lzma_filters") + self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format, + check=check, preset=preset, filters=filters) + + def write(self, data: bytes) -> int: + return self.lzmafile.write(data) + + def close(self) -> None: + self.lzmafile.close() + self.file.close() + + +# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager +# instance as a file like writable object. This could be needed by some exporters +# such as CsvItemExporter which wraps the feed storage with io.TextIOWrapper. +class PostProcessingManager(IOBase): + """ + This will manage and use declared plugins to process data in a + pipeline-ish way. + :param plugins: all the declared plugins for the feed + :type plugins: list + :param file: final target file where the processed data will be written + :type file: file like object + """ + + def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.plugins = self._load_plugins(plugins) + self.file = file + self.feed_options = feed_options + self.head_plugin = self._get_head_plugin() + + def write(self, data: bytes) -> int: + """ + Uses all the declared plugins to process data first, then writes + the processed data to target file. + :param data: data passed to be written to target file + :type data: bytes + :return: returns number of bytes written + :rtype: int + """ + return self.head_plugin.write(data) + + def tell(self) -> int: + return self.file.tell() + + def close(self) -> None: + """ + Close the target file along with all the plugins. + """ + self.head_plugin.close() + + def writable(self) -> bool: + return True + + def _load_plugins(self, plugins: List[Any]) -> List[Any]: + plugins = [load_object(plugin) for plugin in plugins] + return plugins + + def _get_head_plugin(self) -> Any: + prev = self.file + for plugin in self.plugins[::-1]: + prev = plugin(prev, self.feed_options) + return prev diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index bcdbaff24..739e6b958 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -8,6 +8,7 @@ from scrapy import signals from scrapy.mail import MailSender from scrapy.exceptions import NotConfigured + class StatsMailer: def __init__(self, stats, recipients, mail): diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index bf4ae7b45..b43c383fe 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -142,10 +142,6 @@ class WrappedRequest: """ return self.request.meta.get('is_unverifiable', False) - def get_origin_req_host(self): - return urlparse_cached(self.request).hostname - - # python3 uses attributes instead of methods @property def full_url(self): return self.get_full_url() @@ -164,7 +160,7 @@ class WrappedRequest: @property def origin_req_host(self): - return self.get_origin_req_host() + return urlparse_cached(self.request).hostname def has_header(self, name): return name in self.request.headers diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 27bd55c07..89516b9b6 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -6,7 +6,6 @@ See documentation in docs/topics/request-response.rst """ import json -import warnings from contextlib import suppress from typing import Generator, Tuple from urllib.parse import urljoin @@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, http_content_type_encoding, resolve_encoding) from w3lib.html import strip_html5_whitespace -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode @@ -66,13 +64,6 @@ class TextResponse(Response): or self._body_declared_encoding() ) - def body_as_unicode(self): - """Return body as unicode""" - warnings.warn('Response.body_as_unicode() is deprecated, ' - 'please use Response.text instead.', - ScrapyDeprecationWarning, stacklevel=2) - return self.text - def json(self): """ .. versionadded:: 2.2 diff --git a/scrapy/item.py b/scrapy/item.py index 2ccd7ad18..2521ac829 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -9,45 +9,15 @@ from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat from typing import Dict -from warnings import warn -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref -class _BaseItem(object_ref): - """ - Temporary class used internally to avoid the deprecation - warning raised by isinstance checks using BaseItem. - """ - pass - - -class _BaseItemMeta(ABCMeta): - def __instancecheck__(cls, instance): - if cls is BaseItem: - warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__instancecheck__(instance) - - -class BaseItem(_BaseItem, metaclass=_BaseItemMeta): - """ - Deprecated, please use :class:`scrapy.item.Item` instead - """ - - def __new__(cls, *args, **kwargs): - if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): - warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) - - class Field(dict): """Container of field metadata""" -class ItemMeta(_BaseItemMeta): +class ItemMeta(ABCMeta): """Metaclass_ of :class:`Item` that handles field definitions. .. _metaclass: https://realpython.com/python-metaclasses @@ -74,15 +44,30 @@ class ItemMeta(_BaseItemMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class DictItem(MutableMapping, BaseItem): +class Item(MutableMapping, object_ref, metaclass=ItemMeta): + """ + Base class for scraped items. - fields: Dict[str, Field] = {} + In Scrapy, an object is considered an ``item`` if it is an instance of either + :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a + spider callback is evaluated, only instances of :class:`Item` or + :class:`dict` are passed to :ref:`item pipelines `. - def __new__(cls, *args, **kwargs): - if issubclass(cls, DictItem) and not issubclass(cls, Item): - warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) + If you need instances of a custom class to be considered items by Scrapy, + you must inherit from either :class:`Item` or :class:`dict`. + + Items must declare :class:`Field` attributes, which are processed and stored + in the ``fields`` attribute. This restricts the set of allowed field names + and prevents typos, raising ``KeyError`` when referring to undefined fields. + Additionally, fields can be used to define metadata and control the way + data is processed internally. Please refer to the :ref:`documentation + about fields ` for additional information. + + Unlike instances of :class:`dict`, instances of :class:`Item` may be + :ref:`tracked ` to debug memory leaks. + """ + + fields: Dict[str, Field] def __init__(self, *args, **kwargs): self._values = {} @@ -118,7 +103,7 @@ class DictItem(MutableMapping, BaseItem): def __iter__(self): return iter(self._values) - __hash__ = BaseItem.__hash__ + __hash__ = object_ref.__hash__ def keys(self): return self._values.keys() @@ -133,27 +118,3 @@ class DictItem(MutableMapping, BaseItem): """Return a :func:`~copy.deepcopy` of this item. """ return deepcopy(self) - - -class Item(DictItem, metaclass=ItemMeta): - """ - Base class for scraped items. - - In Scrapy, an object is considered an ``item`` if it is an instance of either - :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a - spider callback is evaluated, only instances of :class:`Item` or - :class:`dict` are passed to :ref:`item pipelines `. - - If you need instances of a custom class to be considered items by Scrapy, - you must inherit from either :class:`Item` or :class:`dict`. - - Items must declare :class:`Field` attributes, which are processed and stored - in the ``fields`` attribute. This restricts the set of allowed field names - and prevents typos, raising ``KeyError`` when referring to undefined fields. - Additionally, fields can be used to define metadata and control the way - data is processed internally. Please refer to the :ref:`documentation - about fields ` for additional information. - - Unlike instances of :class:`dict`, instances of :class:`Item` may be - :ref:`tracked ` to debug memory leaks. - """ diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index e941c4321..b5d2585a8 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -88,7 +88,7 @@ class LxmlParserLinkExtractor: def _process_links(self, links): """ Normalize and filter extracted links - The subclass should override it if neccessary + The subclass should override it if necessary """ return self._deduplicate_if_needed(links) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 014951a8e..91337b949 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -83,6 +83,9 @@ class ItemLoader(itemloaders.ItemLoader): def __init__(self, item=None, selector=None, response=None, parent=None, **context): if selector is None and response is not None: - selector = self.default_selector_class(response) + try: + selector = self.default_selector_class(response) + except AttributeError: + selector = None context.update(response=response) super().__init__(item=item, selector=selector, parent=parent, **context) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index bbec38086..2eb1d8609 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict +from typing import Callable, Deque, Dict, Optional, cast, Iterable from twisted.internet.defer import Deferred @@ -9,7 +9,7 @@ from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.defer import process_parallel, process_chain, process_chain_both +from scrapy.utils.defer import process_parallel, process_chain logger = logging.getLogger(__name__) @@ -21,7 +21,8 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods: Dict[str, Deque[Callable]] = defaultdict(deque) + # Optional because process_spider_output and process_spider_exception can be None + self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -64,14 +65,12 @@ class MiddlewareManager: self.methods['close_spider'].appendleft(mw.close_spider) def _process_parallel(self, methodname: str, obj, *args) -> Deferred: - return process_parallel(self.methods[methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_parallel(methods, obj, *args) def _process_chain(self, methodname: str, obj, *args) -> Deferred: - return process_chain(self.methods[methodname], obj, *args) - - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: - return process_chain_both(self.methods[cb_methodname], - self.methods[eb_methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_chain(methods, obj, *args) def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 471c01ead..72e210469 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -85,7 +85,7 @@ class S3FilesStore: AWS_USE_SSL = None AWS_VERIFY = None - POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings + POLICY = 'private' # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings HEADERS = { 'Cache-Control': 'max-age=172800', } @@ -142,7 +142,7 @@ class S3FilesStore: **extra) def _headers_to_botocore_kwargs(self, headers): - """ Convert headers to botocore keyword agruments. + """ Convert headers to botocore keyword arguments. """ # This is required while we need to support both boto and botocore. mapping = CaseInsensitiveDict({ @@ -190,7 +190,7 @@ class GCSFilesStore: CACHE_CONTROL = 'max-age=172800' # The bucket's default object ACL will be applied to the object. - # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. POLICY = None def __init__(self, uri): @@ -291,7 +291,7 @@ class FilesPipeline(MediaPipeline): """Abstract pipeline that implement the file downloading This pipeline tries to minimize network transfers and file processing, - doing stat of the files and determining if file is new, uptodate or + doing stat of the files and determining if file is new, up-to-date or expired. ``new`` files are those that pipeline never processed and needs to be diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 1fe1e6fd1..6b1ad0828 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -375,9 +375,13 @@ class BaseSettings(MutableMapping): return len(self.attributes) def _to_dict(self): - return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) + return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items()} + def _get_key(self, key_value): + return (key_value if isinstance(key_value, (bool, float, int, str, type(None))) + else str(key_value)) + def copy_to_dict(self): """ Make a copy of current settings and convert to a dict. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4ef330dd2..8389a70cb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -207,6 +207,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' LOG_FILE = None +LOG_FILE_APPEND = True LOG_SHORT_NAMES = False SCHEDULER_DEBUG = False diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c13ba4b3c..d8248c606 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -4,14 +4,12 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ import logging -import warnings from typing import Optional from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider -from scrapy.utils.deprecate import method_is_overridden class Spider(object_ref): @@ -57,34 +55,13 @@ class Spider(object_ref): crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self): - cls = self.__class__ if not self.start_urls and hasattr(self, 'start_url'): raise AttributeError( "Crawling could not start: 'start_urls' not found " "or empty (but found 'start_url' attribute instead, " "did you miss an 's'?)") - if method_is_overridden(cls, Spider, 'make_requests_from_url'): - warnings.warn( - "Spider.make_requests_from_url method is deprecated; it " - "won't be called in future Scrapy releases. Please " - "override Spider.start_requests method instead " - f"(see {cls.__module__}.{cls.__name__}).", - ) - for url in self.start_urls: - yield self.make_requests_from_url(url) - else: - for url in self.start_urls: - yield Request(url, dont_filter=True) - - def make_requests_from_url(self, url): - """ This method is deprecated. """ - warnings.warn( - "Spider.make_requests_from_url method is deprecated: " - "it will be removed and not be called by the default " - "Spider.start_requests method in future Scrapy releases. " - "Please override Spider.start_requests method instead." - ) - return Request(url, dont_filter=True) + for url in self.start_urls: + yield Request(url, dont_filter=True) def _parse(self, response, **kwargs): return self.parse(response, **kwargs) diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6ed17e4dd..79e12e030 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -43,7 +43,7 @@ class XMLFeedSpider(Spider): return response def parse_node(self, response, selector): - """This method must be overriden with your custom spider functionality""" + """This method must be overridden with your custom spider functionality""" if hasattr(self, 'parse_item'): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError @@ -113,7 +113,7 @@ class CSVFeedSpider(Spider): return response def parse_row(self, response, row): - """This method must be overriden with your custom spider functionality""" + """This method must be overridden with your custom spider functionality""" raise NotImplementedError def parse_rows(self, response): @@ -123,7 +123,7 @@ class CSVFeedSpider(Spider): process_results methods for pre and post-processing purposes. """ - for row in csviter(response, self.delimiter, self.headers, self.quotechar): + for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar): ret = iterate_spider_output(self.parse_row(response, row)) for result_item in self.process_results(response, ret): yield result_item diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index b904c4a03..24a6187b9 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -121,7 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings): out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) - out.setdefault("item_export_kwargs", dict()) + out.setdefault("item_export_kwargs", {}) if settings["FEED_EXPORT_INDENT"] is None: out.setdefault("indent", None) else: @@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None, message = ( 'The -t command line option is deprecated in favor of ' 'specifying the output format within the output URI. See the ' - 'documentation of the -o and -O options for more information.', + 'documentation of the -o and -O options for more information.' ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 133261fd7..1bc0bd45f 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -14,7 +14,7 @@ def _embed_ipython_shell(namespace={}, banner=''): @wraps(_embed_ipython_shell) def wrapper(namespace=namespace, banner=''): config = load_default_config() - # Always use .instace() to ensure _instance propagation to all parents + # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports # and clear the instance to always have the fresh env # on repeated breaks like with inspect_response() diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 6eeabe1ee..f45e1c9b8 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -57,7 +57,7 @@ class CaselessDict(dict): return key.lower() def normvalue(self, value): - """Method to normalize values prior to be setted""" + """Method to normalize values prior to be set""" return value def get(self, key, def_val=None): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index b317c12a3..7ecb8ea3f 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -3,9 +3,16 @@ Helper functions for dealing with Twisted deferreds """ import asyncio import inspect -from collections.abc import Coroutine +from asyncio import Future from functools import wraps -from typing import Any, Callable, Generator, Iterable +from typing import ( + Any, + Callable, + Coroutine, + Generator, + Iterable, + Union +) from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred @@ -34,7 +41,7 @@ def defer_succeed(result) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop - It delays by 100ms so reactor has a chance to go trough readers and writers + It delays by 100ms so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor @@ -171,3 +178,55 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: return defer.fail(result) else: return defer.succeed(result) + + +def deferred_to_future(d: Deferred) -> Future: + """ + .. versionadded:: 2.6.0 + + Return an :class:`asyncio.Future` object that wraps *d*. + + When :ref:`using the asyncio reactor `, you cannot await + on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy + callables defined as coroutines `, you can only await on + ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects + allows you to wait on them:: + + class MySpider(Spider): + ... + async def parse(self, response): + d = treq.get('https://example.com/additional') + additional_response = await deferred_to_future(d) + """ + return d.asFuture(asyncio.get_event_loop()) + + +def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: + """ + .. versionadded:: 2.6.0 + + Return *d* as an object that can be awaited from a :ref:`Scrapy callable + defined as a coroutine `. + + What you can await in Scrapy callables defined as coroutines depends on the + value of :setting:`TWISTED_REACTOR`: + + - When not using the asyncio reactor, you can only await on + :class:`~twisted.internet.defer.Deferred` objects. + + - When :ref:`using the asyncio reactor `, you can only + await on :class:`asyncio.Future` objects. + + If you want to write code that uses ``Deferred`` objects but works with any + reactor, use this function on all ``Deferred`` objects:: + + class MySpider(Spider): + ... + async def parse(self, response): + d = treq.get('https://example.com/additional') + extra_response = await maybe_deferred_to_future(d) + """ + if not is_asyncio_reactor_installed(): + return d + else: + return deferred_to_future(d) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f5b17416f..ae727464c 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -79,7 +79,7 @@ def create_deprecated_class( # for implementation details def __instancecheck__(cls, inst): return any(cls.__subclasscheck__(c) - for c in {type(inst), inst.__class__}) + for c in (type(inst), inst.__class__)) def __subclasscheck__(cls, sub): if cls is not DeprecatedClass.deprecated_class: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6c456ed60..78e302d19 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -124,8 +124,9 @@ def _get_handler(settings): """ Return a log handler object according to settings """ filename = settings.get('LOG_FILE') if filename: + mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w' encoding = settings.get('LOG_ENCODING') - handler = logging.FileHandler(filename, encoding=encoding) + handler = logging.FileHandler(filename, mode=mode, encoding=encoding) elif settings.getbool('LOG_ENABLED'): handler = logging.StreamHandler() else: @@ -142,7 +143,7 @@ def _get_handler(settings): return handler -def log_scrapy_info(settings): +def log_scrapy_info(settings: Settings) -> None: logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ @@ -151,6 +152,9 @@ def log_scrapy_info(settings): if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) + + +def log_reactor_info() -> None: from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) from twisted.internet import asyncioreactor diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 51cef1e91..1221b39b2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -14,11 +14,11 @@ from w3lib.html import replace_entities from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.python import flatten, to_unicode -from scrapy.item import _BaseItem +from scrapy.item import Item from scrapy.utils.deprecate import ScrapyDeprecationWarning -_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes +_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes def arg_to_iter(arg): @@ -50,7 +50,7 @@ def load_object(path): return path else: raise TypeError("Unexpected argument type, expected string " - "or object, got: %s" % type(path)) + f"or object, got: {type(path)}") try: dot = path.rindex('.') diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6723d9b37..96395543c 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,4 +1,5 @@ import asyncio +import sys from contextlib import suppress from twisted.internet import asyncioreactor, error @@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): + if sys.version_info >= (3, 8) and sys.platform == "win32": + policy = asyncio.get_event_loop_policy() + if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 57dcc5f2c..70ef3ba2b 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -48,7 +48,7 @@ def request_fingerprint( the fingerprint. For this reason, request headers are ignored by default when calculating - the fingeprint. If you want to include specific headers use the + the fingerprint. If you want to include specific headers use the include_headers argument, which is a list of Request headers to include. Also, servers usually ignore fragments in urls when handling requests, @@ -78,7 +78,7 @@ def request_fingerprint( def request_authenticate(request: Request, username: str, password: str) -> None: - """Autenticate the given request (in place) using the HTTP basic access + """Authenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ request.headers['Authorization'] = basic_auth_header(username, password) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b3ef7b463..741dce350 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,8 +3,9 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import webbrowser +import re import tempfile +import webbrowser from typing import Any, Callable, Iterable, Optional, Tuple, Union from weakref import WeakKeyDictionary @@ -13,6 +14,7 @@ from scrapy.http.response import Response from twisted.web import http from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.decorators import deprecated from w3lib import html @@ -50,6 +52,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str: return f'{status_int} {to_unicode(message)}' +@deprecated def response_httprepr(response: Response) -> bytes: """Return raw HTTP representation (as bytes) of the given response. This is provided only for reference, since it's not the exact stream of bytes @@ -80,8 +83,9 @@ def open_in_browser( body = response.body if isinstance(response, HtmlResponse): if b'' - body = body.replace(b'', to_bytes(repl)) + repl = fr'\1' + body = re.sub(b"", b"", body, flags=re.DOTALL) + body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 00226283f..f704e113f 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -260,7 +260,7 @@ ItemForm ia['width'] = x.x('//p[@class="width"]') ia['volume'] = x.x('//p[@class="volume"]') - # another example passing parametes on instance + # another example passing parameters on instance ia = NewsForm(response, encoding='utf-8') ia['name'] = x.x('//p[@class="name"]') diff --git a/sep/sep-005.rst b/sep/sep-005.rst index e795838e4..08ed367b3 100644 --- a/sep/sep-005.rst +++ b/sep/sep-005.rst @@ -107,7 +107,7 @@ gUsing default_builder This will use default_builder as the builder for every field in the item class. -As a reducer is not set reducers will be set based on Item Field classess. +As a reducer is not set reducers will be set based on Item Field classes. gReset default_builder for a field ================================== diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 8ca81824d..0859e3f7c 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -64,7 +64,7 @@ Request Processors takes requests objects and can perform any action to them, like filtering or modifying on the fly. The current ``LinkExtractor`` had integrated link processing, like -canonicalize. Request Processors can be reutilized and applied in serie. +canonicalize. Request Processors can be reutilized and applied in series. Request Generator ----------------- diff --git a/sep/sep-021.rst b/sep/sep-021.rst index 372429791..c1ec16f7f 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -22,7 +22,7 @@ Instead, the hooks are spread over: * Downloader handlers (DOWNLOADER_HANDLERS) * Item pipelines (ITEM_PIPELINES) * Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES) -* Overrideable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) +* Overridable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) * Generic extensions (EXTENSIONS) * CLI commands (COMMANDS_MODULE) diff --git a/setup.py b/setup.py index ed2b6e347..d86c0f285 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'setuptools', + 'tldextract', ] extras_require = {} cpython_dependencies = [ @@ -87,6 +88,7 @@ setup( 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 8568bd8b8..f2a93074b 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,6 +1,9 @@ import asyncio +import sys from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) import scrapy diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..56304bd23 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py new file mode 100644 index 000000000..3f219098c --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class SelectReactorSpider(scrapy.Spider): + name = 'select_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(SelectReactorSpider) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..72bb986bc --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = 'asyncio_reactor1' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +class AsyncioReactorSpider2(scrapy.Spider): + name = 'asyncio_reactor2' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index da202be4d..bb4a8e5af 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -40,9 +40,9 @@ def generate_keys(): subject = issuer = Name( [ - NameAttribute(NameOID.COUNTRY_NAME, u"IE"), - NameAttribute(NameOID.ORGANIZATION_NAME, u"Scrapy"), - NameAttribute(NameOID.COMMON_NAME, u"localhost"), + NameAttribute(NameOID.COUNTRY_NAME, "IE"), + NameAttribute(NameOID.ORGANIZATION_NAME, "Scrapy"), + NameAttribute(NameOID.COMMON_NAME, "localhost"), ] ) cert = ( @@ -54,7 +54,7 @@ def generate_keys(): .not_valid_before(datetime.utcnow()) .not_valid_after(datetime.utcnow() + timedelta(days=10)) .add_extension( - SubjectAlternativeName([DNSName(u"localhost")]), + SubjectAlternativeName([DNSName("localhost")]), critical=False, ) .sign(key, SHA256(), default_backend()) diff --git a/tests/mockserver.py b/tests/mockserver.py index ab9aec6a6..72d7e0241 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -14,10 +14,9 @@ from twisted.internet import defer, reactor, ssl from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory -from twisted.web.resource import EncodingResourceWrapper, Resource +from twisted.web import resource, server from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site from twisted.web.static import File -from twisted.web.test.test_webclient import PayloadResource from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode @@ -35,7 +34,70 @@ def getarg(request, name, default=None, type=None): return default -class LeafResource(Resource): +# most of the following resources are copied from twisted.web.test.test_webclient +class ForeverTakingResource(resource.Resource): + """ + L{ForeverTakingResource} is a resource which never finishes responding + to requests. + """ + + def __init__(self, write=False): + resource.Resource.__init__(self) + self._write = write + + def render(self, request): + if self._write: + request.write(b"some bytes") + return server.NOT_DONE_YET + + +class ErrorResource(resource.Resource): + def render(self, request): + request.setResponseCode(401) + if request.args.get(b"showlength"): + request.setHeader(b"content-length", b"0") + return b"" + + +class NoLengthResource(resource.Resource): + def render(self, request): + return b"nolength" + + +class HostHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the host header + from the request. + """ + + def render(self, request): + return request.requestHeaders.getRawHeaders(b"host")[0] + + +class PayloadResource(resource.Resource): + """ + A testing resource which renders itself as the contents of the request body + as long as the request body is 100 bytes long, otherwise which renders + itself as C{"ERROR"}. + """ + + def render(self, request): + data = request.content.read() + contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] + if len(data) != 100 or int(contentLength) != 100: + return b"ERROR" + return data + + +class BrokenDownloadResource(resource.Resource): + def render(self, request): + # only sends 3 bytes even though it claims to send 5 + request.setHeader(b"content-length", b"5") + request.write(b"abc") + return b"" + + +class LeafResource(resource.Resource): isLeaf = True @@ -175,10 +237,10 @@ class ArbitraryLengthPayloadResource(LeafResource): return request.content.read() -class Root(Resource): +class Root(resource.Resource): def __init__(self): - Resource.__init__(self) + resource.Resource.__init__(self) self.putChild(b"status", Status()) self.putChild(b"follow", Follow()) self.putChild(b"delay", Delay()) @@ -187,7 +249,7 @@ class Root(Resource): self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) self.putChild(b"payload", PayloadResource()) - self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"xpayload", resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) try: from tests import tests_datadir diff --git a/tests/requirements.txt b/tests/requirements.txt index bd72c8c46..d2a8aae1b 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,7 +3,7 @@ attrs dataclasses; python_version == '3.6' pyftpdlib pytest -pytest-cov +pytest-cov==3.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures @@ -12,7 +12,7 @@ uvloop; platform_system != "Windows" and python_version > '3.6' # optional for shell wrapper tests bpython -brotlipy # optional for HTTP compress downloader middleware tests +brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tests/spiders.py b/tests/spiders.py index 5b45f897e..67dbbbe0f 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -86,7 +86,7 @@ class SimpleSpider(MetaSpider): self.start_urls = [url] def parse(self, response): - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefSpider(SimpleSpider): @@ -95,7 +95,7 @@ class AsyncDefSpider(SimpleSpider): async def parse(self, response): await defer.succeed(42) - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioSpider(SimpleSpider): @@ -105,7 +105,7 @@ class AsyncDefAsyncioSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") class AsyncDefAsyncioReturnSpider(SimpleSpider): @@ -115,7 +115,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") return [{'id': 1}, {'id': 2}] @@ -126,7 +126,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.1) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") return {"foo": 42} @@ -138,7 +138,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): await asyncio.sleep(0.2) req_id = response.meta.get('req_id', 0) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d, req_id %d" % (status, req_id)) + self.logger.info(f"Got response {status}, req_id {req_id}") if req_id > 0: return reqs = [] @@ -155,7 +155,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) yield {'foo': 42} - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenLoopSpider(SimpleSpider): @@ -166,7 +166,7 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider): for i in range(10): await asyncio.sleep(0.1) yield {'foo': i} - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenComplexSpider(SimpleSpider): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 5ec5e2989..be8adadb3 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -41,7 +41,7 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') - key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__) + key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}' errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 591075a98..8233e0101 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -64,3 +64,7 @@ class CmdlineTest(unittest.TestCase): settingsdict = json.loads(settingsstr) self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) self.assertEqual(200, settingsdict[EXT_PATH]) + + def test_pathlib_path_as_feeds_key(self): + self.assertEqual(self._execute('settings', '--get', 'FEEDS'), + json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}})) diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py index 8a719ddf2..b0ac6e98b 100644 --- a/tests/test_cmdline/settings.py +++ b/tests/test_cmdline/settings.py @@ -1,5 +1,14 @@ +from pathlib import Path + EXTENSIONS = { 'tests.test_cmdline.extensions.TestExtension': 0, } TEST1 = 'default' + +FEEDS = { + Path('items.csv'): { + 'format': 'csv', + 'fields': ['price', 'name'], + }, +} diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 34f5e59dd..c3d705194 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -19,11 +19,11 @@ import scrapy class CheckSpider(scrapy.Spider): name = '{self.spider_name}' - start_urls = ['http://example.com'] + start_urls = ['http://toscrape.com'] def parse(self, response, **cb_kwargs): \"\"\" - @url http://example.com + @url http://toscrape.com {contracts} \"\"\" {parse_def} diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index ed3848d88..f21ee971d 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,6 +1,9 @@ import os +import argparse from os.path import join, abspath, isfile, exists from twisted.internet import defer +from scrapy.commands import parse +from scrapy.settings import Settings from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest from scrapy.utils.python import to_unicode @@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} content = '[\n{},\n{"foo": "bar"}\n]' with open(file_path, 'r') as f: self.assertEqual(f.read(), content) + + def test_parse_add_options(self): + command = parse.Command() + command.settings = Settings() + parser = argparse.ArgumentParser( + prog='scrapy', formatter_class=argparse.HelpFormatter, + conflict_handler='resolve', prefix_chars='-' + ) + command.add_options(parser) + namespace = parser.parse_args( + ['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name] + ) + self.assertTrue(namespace.nolinks) + self.assertEqual(namespace.depth, 2) + self.assertEqual(namespace.spider, self.spider_name) + self.assertTrue(namespace.verbose) diff --git a/tests/test_commands.py b/tests/test_commands.py index eec1f02ee..7cd19b29a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,8 +1,9 @@ import inspect import json -import optparse +import argparse import os import platform +import re import subprocess import sys import tempfile @@ -22,7 +23,7 @@ from twisted.python.versions import Version from twisted.trial import unittest import scrapy -from scrapy.commands import ScrapyCommand +from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -36,19 +37,28 @@ class CommandSettings(unittest.TestCase): def setUp(self): self.command = ScrapyCommand() self.command.settings = Settings() - self.parser = optparse.OptionParser( - formatter=optparse.TitledHelpFormatter(), - conflict_handler='resolve', - ) + self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, + conflict_handler='resolve') self.command.add_options(self.parser) def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' - opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) + opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) + def test_help_formatter(self): + formatter = ScrapyHelpFormatter(prog='scrapy') + part_strings = ['usage: scrapy genspider [options] \n\n', + '\n', 'optional arguments:\n', '\n', 'Global Options:\n'] + self.assertEqual( + formatter._join_parts(part_strings), + ('Usage\n=====\n scrapy genspider [options] \n\n\n' + 'Optional Arguments\n==================\n\n' + 'Global Options\n--------------\n') + ) + class ProjectTest(unittest.TestCase): project_name = 'testproject' @@ -71,9 +81,14 @@ class ProjectTest(unittest.TestCase): def proc(self, *new_args, **popen_kwargs): args = (sys.executable, '-m', 'scrapy.cmdline') + new_args - p = subprocess.Popen(args, cwd=self.cwd, env=self.env, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **popen_kwargs) + p = subprocess.Popen( + args, + cwd=popen_kwargs.pop('cwd', self.cwd), + env=self.env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **popen_kwargs, + ) def kill_proc(): p.kill() @@ -89,11 +104,23 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) + def find_in_file(self, filename, regex): + """Find first pattern occurrence in file""" + pattern = re.compile(regex) + with open(filename, "r") as f: + for line in f: + match = pattern.search(line) + if match is not None: + return match + class StartprojectTest(ProjectTest): def test_startproject(self): - self.assertEqual(0, self.call('startproject', self.project_name)) + p, out, err = self.proc('startproject', self.project_name) + print(out) + print(err, file=sys.stderr) + self.assertEqual(p.returncode, 0) assert exists(join(self.proj_path, 'scrapy.cfg')) assert exists(join(self.proj_path, 'testproject')) @@ -128,6 +155,25 @@ class StartprojectTest(ProjectTest): self.assertEqual(2, self.call('startproject')) self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) + def test_existing_project_dir(self): + project_dir = mkdtemp() + project_name = self.project_name + '_existing' + project_path = os.path.join(project_dir, project_name) + os.mkdir(project_path) + + p, out, err = self.proc('startproject', project_name, cwd=project_dir) + print(out) + print(err, file=sys.stderr) + self.assertEqual(p.returncode, 0) + + assert exists(join(abspath(project_path), 'scrapy.cfg')) + assert exists(join(abspath(project_path), project_name)) + assert exists(join(join(abspath(project_path), project_name), '__init__.py')) + assert exists(join(join(abspath(project_path), project_name), 'items.py')) + assert exists(join(join(abspath(project_path), project_name), 'pipelines.py')) + assert exists(join(join(abspath(project_path), project_name), 'settings.py')) + assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py')) + def get_permissions_dict(path, renamings=None, ignore=None): @@ -455,6 +501,26 @@ class GenspiderCommandTest(CommandTest): def test_same_filename_as_existing_spider_force(self): self.test_same_filename_as_existing_spider(force=True) + def test_url(self, url='test.com', domain="test.com"): + self.assertEqual(0, self.call('genspider', '--force', 'test_name', url)) + self.assertEqual(domain, + self.find_in_file(join(self.proj_mod_path, + 'spiders', 'test_name.py'), + r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) + self.assertEqual(f'http://{domain}/', + self.find_in_file(join(self.proj_mod_path, + 'spiders', 'test_name.py'), + r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) + + def test_url_schema(self): + self.test_url('http://test.com', 'test.com') + + def test_url_path(self): + self.test_url('test.com/some/other/page', 'test.com') + + def test_url_schema_path(self): + self.test_url('https://test.com/some/other/page', 'test.com') + class GenspiderStandaloneCommandTest(ProjectTest): @@ -617,9 +683,6 @@ class MySpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' @@ -642,16 +705,16 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_custom_asyncio_loop_enabled_false(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' ]) import asyncio - loop = asyncio.new_event_loop() - self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log) + if sys.platform != 'win32': + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() + self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) def test_output(self): spider_code = """ @@ -708,6 +771,7 @@ class MySpider(scrapy.Spider): self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) +@skipIf(platform.system() != 'Windows', "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = 'myspider.pyw' @@ -720,35 +784,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_run_good_spider(self): super().test_run_good_spider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider(self): super().test_runspider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_dnscache_disabled(self): super().test_runspider_dnscache_disabled() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_level(self): super().test_runspider_log_level() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_short_names(self): super().test_runspider_log_short_names() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_no_spider_found(self): super().test_runspider_no_spider_found() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_output(self): super().test_output() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_overwrite_output(self): super().test_overwrite_output() @@ -765,6 +821,21 @@ class BenchCommandTest(CommandTest): self.assertNotIn('Unhandled Error', log) +class ViewCommandTest(CommandTest): + + def test_methods(self): + command = view.Command() + command.settings = Settings() + parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-', + formatter_class=ScrapyHelpFormatter, + conflict_handler='resolve') + command.add_options(parser) + self.assertEqual(command.short_desc(), + "Open URL in browser, as seen by Scrapy") + self.assertIn("URL using the Scrapy downloader and show its", + command.long_desc()) + + class CrawlCommandTest(CommandTest): def crawl(self, code, args=()): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 84bac9b50..7bda3bef2 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -274,6 +274,28 @@ with multiples lines self.assertEqual(s['engine.spider.name'], crawler.spider.name) self.assertEqual(s['len(engine.scraper.slot.active)'], 1) + @defer.inlineCallbacks + def test_format_engine_status(self): + from scrapy.utils.engine import format_engine_status + est = [] + + def cb(response): + est.append(format_engine_status(crawler.engine)) + + crawler = self.runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) + self.assertEqual(len(est), 1, est) + est = est[0].split("\n")[2:-2] # remove header & footer + # convert to dict + est = [x.split(":") for x in est] + est = [x for sublist in est for x in sublist] # flatten + est = [x.lstrip().rstrip() for x in est] + it = iter(est) + s = dict(zip(it, it)) + + self.assertEqual(s['engine.spider.name'], crawler.spider.name) + self.assertEqual(s['len(engine.scraper.slot.active)'], '1') + @defer.inlineCallbacks def test_graceful_crawl_error_handling(self): """ diff --git a/tests/test_crawler.py b/tests/test_crawler.py index dec517bb6..8f6227109 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,10 +4,8 @@ import platform import subprocess import sys import warnings -from unittest import skipIf from pytest import raises, mark -from testfixtures import LogCapture from twisted import version as twisted_version from twisted.internet import defer from twisted.python.versions import Version @@ -98,6 +96,8 @@ class CrawlerLoggingTestCase(unittest.TestCase): def test_spider_custom_settings_log_level(self): log_file = self.mktemp() + with open(log_file, 'wb') as fo: + fo.write('previous message\n'.encode('utf-8')) class MySpider(scrapy.Spider): name = 'spider' @@ -119,8 +119,9 @@ class CrawlerLoggingTestCase(unittest.TestCase): logging.error('error message') with open(log_file, 'rb') as fo: - logged = fo.read().decode('utf8') + logged = fo.read().decode('utf-8') + self.assertIn('previous message', logged) self.assertNotIn('debug message', logged) self.assertIn('info message', logged) self.assertIn('warning message', logged) @@ -131,6 +132,30 @@ class CrawlerLoggingTestCase(unittest.TestCase): crawler.stats.get_value('log_count/INFO') - info_count, 1) self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) + def test_spider_custom_settings_log_append(self): + log_file = self.mktemp() + with open(log_file, 'wb') as fo: + fo.write('previous message\n'.encode('utf-8')) + + class MySpider(scrapy.Spider): + name = 'spider' + custom_settings = { + 'LOG_FILE': log_file, + 'LOG_FILE_APPEND': False, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, + } + + configure_logging() + Crawler(MySpider, {}) + logging.debug('debug message') + + with open(log_file, 'rb') as fo: + logged = fo.read().decode('utf-8') + + self.assertNotIn('previous message', logged) + self.assertIn('debug message', logged) + class SpiderLoaderWithWrongInterface: @@ -244,6 +269,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertEqual(runner.bootstrap_failed, True) + @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ @@ -252,35 +278,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - CrawlerRunner(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - - @defer.inlineCallbacks - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") - def test_crawler_process_asyncio_enabled_true(self): - with LogCapture(level=logging.DEBUG) as log: - if self.reactor_pytest == 'asyncio': - runner = CrawlerProcess(settings={ + runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) yield runner.crawl(NoRequestsSpider) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) - else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - - @defer.inlineCallbacks - def test_crawler_process_asyncio_enabled_false(self): - runner = CrawlerProcess(settings={"TWISTED_REACTOR": None}) - with LogCapture(level=logging.DEBUG) as log: - yield runner.crawl(NoRequestsSpider) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) class ScriptRunnerMixin: @@ -301,17 +302,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -350,14 +345,26 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + def test_reactor_asyncio_custom_settings(self): + log = self.run_script("twisted_reactor_custom_settings.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_reactor_asyncio_custom_settings_same(self): + log = self.run_script("twisted_reactor_custom_settings_same.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') @@ -377,9 +384,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 9c11820e5..2bb53950d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,6 +1,7 @@ import contextlib import os import shutil +import sys import tempfile from typing import Optional, Type from unittest import mock @@ -14,8 +15,6 @@ from twisted.trial import unittest from twisted.web import resource, server, static, util from twisted.web._newclient import ResponseFailed from twisted.web.http import _DataLoss -from twisted.web.test.test_webclient import (ForeverTakingResource, HostHeaderResource, - NoLengthResource, PayloadResource) from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers @@ -33,7 +32,15 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto -from tests.mockserver import MockServer, ssl_context_factory, Echo +from tests.mockserver import ( + Echo, + ForeverTakingResource, + HostHeaderResource, + MockServer, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) from tests.spiders import SingleRequestSpider @@ -287,6 +294,12 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects but no data is received spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -296,6 +309,11 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) # client connects, server send headers and some body bytes but hangs spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -1055,6 +1073,10 @@ class BaseFTPTestCase(unittest.TestCase): class FTPTestCase(BaseFTPTestCase): def test_invalid_credentials(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) from twisted.protocols.ftp import ConnectionLost meta = dict(self.req_meta) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 53bb4fe92..3a9db3ee5 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -219,7 +219,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): certfile = 'keys/localhost.crt' scheme = 'https' - host = u'127.0.0.1' + host = '127.0.0.1' expected_http_proxy_request_body = b'/' @@ -248,7 +248,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, b'/') - http_proxy = '%s?noconnect' % self.getURL('') + http_proxy = f"{self.getURL('')}?noconnect" request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex( Warning, diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index aff8542e9..ba7453255 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -6,13 +6,57 @@ import pytest from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler +def _cookie_to_set_cookie_value(cookie): + """Given a cookie defined as a dictionary with name and value keys, and + optional path and domain keys, return the equivalent string that can be + associated to a ``Set-Cookie`` header.""" + decoded = {} + for key in ("name", "value", "path", "domain"): + if cookie.get(key) is None: + if key in ("name", "value"): + return + continue + if isinstance(cookie[key], (bool, float, int, str)): + decoded[key] = str(cookie[key]) + else: + try: + decoded[key] = cookie[key].decode("utf8") + except UnicodeDecodeError: + decoded[key] = cookie[key].decode("latin1", errors="replace") + + cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" + for key, value in decoded.items(): # path, domain + cookie_str += f"; {key.capitalize()}={value}" + return cookie_str + + +def _cookies_to_set_cookie_list(cookies): + """Given a group of cookie defined either as a dictionary or as a list of + dictionaries (i.e. in a format supported by the cookies parameter of + Request), return the equivalen list of strings that can be associated to a + ``Set-Cookie`` header.""" + if not cookies: + return [] + if isinstance(cookies, dict): + cookies = ({"name": k, "value": v} for k, v in cookies.items()) + return filter( + None, + ( + _cookie_to_set_cookie_value(cookie) + for cookie in cookies + ) + ) + + class CookiesMiddlewareTest(TestCase): def assertCookieValEqual(self, first, second, msg=None): @@ -23,9 +67,11 @@ class CookiesMiddlewareTest(TestCase): def setUp(self): self.spider = Spider('foo') self.mw = CookiesMiddleware() + self.redirect_middleware = RedirectMiddleware(settings=Settings()) def tearDown(self): del self.mw + del self.redirect_middleware def test_basic(self): req = Request('http://scrapytest.org/') @@ -347,3 +393,303 @@ class CookiesMiddlewareTest(TestCase): self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') self.assertCookieValEqual(req3.headers['Cookie'], 'key=') + + def test_primitive_type_cookies(self): + # Boolean + req1 = Request('http://example.org', cookies={'a': True}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'a=True') + + # Float + req2 = Request('http://example.org', cookies={'a': 9.5}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'a=9.5') + + # Integer + req3 = Request('http://example.org', cookies={'a': 10}) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], b'a=10') + + # String + req4 = Request('http://example.org', cookies={'a': 'b'}) + assert self.mw.process_request(req4, self.spider) is None + self.assertCookieValEqual(req4.headers['Cookie'], b'a=b') + + def _test_cookie_redirect( + self, + source, + target, + *, + cookies1, + cookies2, + ): + input_cookies = {'a': 'b'} + + if not isinstance(source, dict): + source = {'url': source} + if not isinstance(target, dict): + target = {'url': target} + target.setdefault('status', 301) + + request1 = Request(cookies=input_cookies, **source) + self.mw.process_request(request1, self.spider) + cookies = request1.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies1 else None) + + response = Response( + headers={ + 'Location': target['url'], + }, + **target, + ) + self.assertEqual( + self.mw.process_response(request1, response, self.spider), + response, + ) + + request2 = self.redirect_middleware.process_response( + request1, + response, + self.spider, + ) + self.assertIsInstance(request2, Request) + + self.mw.process_request(request2, self.spider) + cookies = request2.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + def test_cookie_redirect_same_domain(self): + self._test_cookie_redirect( + 'https://toscrape.com', + 'https://toscrape.com', + cookies1=True, + cookies2=True, + ) + + def test_cookie_redirect_same_domain_forcing_get(self): + self._test_cookie_redirect( + 'https://toscrape.com', + {'url': 'https://toscrape.com', 'status': 302}, + cookies1=True, + cookies2=True, + ) + + def test_cookie_redirect_different_domain(self): + self._test_cookie_redirect( + 'https://toscrape.com', + 'https://example.com', + cookies1=True, + cookies2=False, + ) + + def test_cookie_redirect_different_domain_forcing_get(self): + self._test_cookie_redirect( + 'https://toscrape.com', + {'url': 'https://example.com', 'status': 302}, + cookies1=True, + cookies2=False, + ) + + def _test_cookie_header_redirect( + self, + source, + target, + *, + cookies2, + ): + """Test the handling of a user-defined Cookie header when building a + redirect follow-up request. + + We follow RFC 6265 for cookie handling. The Cookie header can only + contain a list of key-value pairs (i.e. no additional cookie + parameters like Domain or Path). Because of that, we follow the same + rules that we would follow for the handling of the Set-Cookie response + header when the Domain is not set: the cookies must be limited to the + target URL domain (not even subdomains can receive those cookies). + + .. note:: This method tests the scenario where the cookie middleware is + disabled. Because of known issue #1992, when the cookies + middleware is enabled we do not need to be concerned about + the Cookie header getting leaked to unintended domains, + because the middleware empties the header from every request. + """ + if not isinstance(source, dict): + source = {'url': source} + if not isinstance(target, dict): + target = {'url': target} + target.setdefault('status', 301) + + request1 = Request(headers={'Cookie': b'a=b'}, **source) + + response = Response( + headers={ + 'Location': target['url'], + }, + **target, + ) + + request2 = self.redirect_middleware.process_response( + request1, + response, + self.spider, + ) + self.assertIsInstance(request2, Request) + + cookies = request2.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + def test_cookie_header_redirect_same_domain(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + 'https://toscrape.com', + cookies2=True, + ) + + def test_cookie_header_redirect_same_domain_forcing_get(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + {'url': 'https://toscrape.com', 'status': 302}, + cookies2=True, + ) + + def test_cookie_header_redirect_different_domain(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + 'https://example.com', + cookies2=False, + ) + + def test_cookie_header_redirect_different_domain_forcing_get(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + {'url': 'https://example.com', 'status': 302}, + cookies2=False, + ) + + def _test_user_set_cookie_domain_followup( + self, + url1, + url2, + domain, + *, + cookies1, + cookies2, + ): + input_cookies = [ + { + 'name': 'a', + 'value': 'b', + 'domain': domain, + } + ] + + request1 = Request(url1, cookies=input_cookies) + self.mw.process_request(request1, self.spider) + cookies = request1.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies1 else None) + + request2 = Request(url2) + self.mw.process_request(request2, self.spider) + cookies = request2.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + def test_user_set_cookie_domain_suffix_private(self): + self._test_user_set_cookie_domain_followup( + 'https://books.toscrape.com', + 'https://quotes.toscrape.com', + 'toscrape.com', + cookies1=True, + cookies2=True, + ) + + def test_user_set_cookie_domain_suffix_public_period(self): + self._test_user_set_cookie_domain_followup( + 'https://foo.co.uk', + 'https://bar.co.uk', + 'co.uk', + cookies1=False, + cookies2=False, + ) + + def test_user_set_cookie_domain_suffix_public_private(self): + self._test_user_set_cookie_domain_followup( + 'https://foo.blogspot.com', + 'https://bar.blogspot.com', + 'blogspot.com', + cookies1=False, + cookies2=False, + ) + + def test_user_set_cookie_domain_public_period(self): + self._test_user_set_cookie_domain_followup( + 'https://co.uk', + 'https://co.uk', + 'co.uk', + cookies1=True, + cookies2=True, + ) + + def _test_server_set_cookie_domain_followup( + self, + url1, + url2, + domain, + *, + cookies, + ): + request1 = Request(url1) + self.mw.process_request(request1, self.spider) + + input_cookies = [ + { + 'name': 'a', + 'value': 'b', + 'domain': domain, + } + ] + + headers = { + 'Set-Cookie': _cookies_to_set_cookie_list(input_cookies), + } + response = Response(url1, status=200, headers=headers) + self.assertEqual( + self.mw.process_response(request1, response, self.spider), + response, + ) + + request2 = Request(url2) + self.mw.process_request(request2, self.spider) + actual_cookies = request2.headers.get('Cookie') + self.assertEqual(actual_cookies, b"a=b" if cookies else None) + + def test_server_set_cookie_domain_suffix_private(self): + self._test_server_set_cookie_domain_followup( + 'https://books.toscrape.com', + 'https://quotes.toscrape.com', + 'toscrape.com', + cookies=True, + ) + + def test_server_set_cookie_domain_suffix_public_period(self): + self._test_server_set_cookie_domain_followup( + 'https://foo.co.uk', + 'https://bar.co.uk', + 'co.uk', + cookies=False, + ) + + def test_server_set_cookie_domain_suffix_public_private(self): + self._test_server_set_cookie_domain_followup( + 'https://foo.blogspot.com', + 'https://bar.blogspot.com', + 'blogspot.com', + cookies=False, + ) + + def test_server_set_cookie_domain_public_period(self): + self._test_server_set_cookie_domain_followup( + 'https://co.uk', + 'https://co.uk', + 'co.uk', + cookies=True, + ) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 3381632b0..0362e2018 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,13 +1,60 @@ import unittest +from w3lib.http import basic_auth_header + from scrapy.http import Request from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spiders import Spider +class TestSpiderLegacy(Spider): + http_user = 'foo' + http_pass = 'bar' + + class TestSpider(Spider): http_user = 'foo' http_pass = 'bar' + http_auth_domain = 'example.com' + + +class TestSpiderAny(Spider): + http_user = 'foo' + http_pass = 'bar' + http_auth_domain = None + + +class HttpAuthMiddlewareLegacyTest(unittest.TestCase): + + def setUp(self): + self.spider = TestSpiderLegacy('foo') + + def test_auth(self): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) + + # initial request, sets the domain and sends the header + req = Request('http://example.com/') + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + # subsequent request to the same domain, should send the header + req = Request('http://example.com/') + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + # subsequent request to a different domain, shouldn't send the header + req = Request('http://example-noauth.com/') + assert mw.process_request(req, self.spider) is None + self.assertNotIn('Authorization', req.headers) + + def test_auth_already_set(self): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) + req = Request('http://example.com/', + headers=dict(Authorization='Digest 123')) + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') class HttpAuthMiddlewareTest(unittest.TestCase): @@ -20,13 +67,45 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def tearDown(self): del self.mw - def test_auth(self): - req = Request('http://scrapytest.org/') + def test_no_auth(self): + req = Request('http://example-noauth.com/') assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') + self.assertNotIn('Authorization', req.headers) + + def test_auth_domain(self): + req = Request('http://example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + def test_auth_subdomain(self): + req = Request('http://foo.example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) def test_auth_already_set(self): - req = Request('http://scrapytest.org/', + req = Request('http://example.com/', + headers=dict(Authorization='Digest 123')) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') + + +class HttpAuthAnyMiddlewareTest(unittest.TestCase): + + def setUp(self): + self.mw = HttpAuthMiddleware() + self.spider = TestSpiderAny('foo') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_auth(self): + req = Request('http://example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + def test_auth_already_set(self): + req = Request('http://example.com/', headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers['Authorization'], b'Digest 123') diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 1f2616e35..9e75f0a50 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,8 +1,10 @@ +from itertools import product from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.response import response_httprepr from scrapy.utils.test import get_crawler @@ -37,6 +39,23 @@ class TestDownloaderStats(TestCase): self.mw.process_response(self.req, self.res, self.spider) self.assertStatsEqual('downloader/response_count', 1) + def test_response_len(self): + body = (b'', b'not_empty') # empty/notempty body + headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'}) # 0 headers, 1h and 2h + test_responses = [ # form test responses with all combinations of body/headers + Response( + url='scrapytest.org', + status=200, + body=r[0], + headers=r[1] + ) + for r in product(body, headers) + ] + for test_response in test_responses: + self.crawler.stats.set_value('downloader/response_bytes', 0) + self.mw.process_response(self.req, test_response, self.spider) + self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response))) + def test_process_exception(self): self.mw.process_exception(self.req, MyException(), self.spider) self.assertStatsEqual('downloader/exception_count', 1) diff --git a/tests/test_engine.py b/tests/test_engine.py index 92bf45f25..fa7d0c8d4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -152,7 +152,7 @@ class CrawlerRun: self.itemerror = [] self.itemresp = [] self.headers = {} - self.bytes = defaultdict(lambda: list()) + self.bytes = defaultdict(list) self.signals_caught = {} self.spider_class = spider_class diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 04bae31d3..b263b3475 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -362,14 +362,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item=dict(text=u'W\u0275\u200Brd'), + item=dict(text='W\u0275\u200Brd'), expected=None, encoding='windows-1251', ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text=u'W\u0275\u200Brd'), + item=dict(text='W\u0275\u200Brd'), include_headers_line=False, expected='Wɵ​rd\r\n', encoding='windows-1251', diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 53e6a2018..f0acf1941 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,5 +1,8 @@ +import bz2 import csv +import gzip import json +import lzma import os import random import shutil @@ -1473,6 +1476,499 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(row['expected'], data[feed_options['format']]) +class FeedPostProcessedExportsTest(FeedExportTestBase): + __test__ = True + + items = [{'foo': 'bar'}] + expected = b'foo\r\nbar\r\n' + + class MyPlugin1: + def __init__(self, file, feed_options): + self.file = file + self.feed_options = feed_options + self.char = self.feed_options.get('plugin1_char', b'') + + def write(self, data): + written_count = self.file.write(data) + written_count += self.file.write(self.char) + return written_count + + def close(self): + self.file.close() + + def _named_tempfile(self, name): + return os.path.join(self.temp_dir, name) + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings): + """ Run spider with specified settings; return exported data with filename. """ + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() + } + + content = {} + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + spider_cls.start_urls = [s.url('/')] + yield runner.crawl(spider_cls) + + for file_path, feed_options in FEEDS.items(): + if not os.path.exists(str(file_path)): + continue + + with open(str(file_path), 'rb') as f: + content[str(file_path)] = f.read() + + finally: + for file_path in FEEDS.keys(): + if not os.path.exists(str(file_path)): + continue + + os.remove(str(file_path)) + + return content + + def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''): + data_stream = BytesIO() + gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime, + compresslevel=compresslevel, mode="wb") + gzipf.write(data) + gzipf.close() + data_stream.seek(0) + return data_stream.read() + + @defer.inlineCallbacks + def test_gzip_plugin(self): + + filename = self._named_tempfile('gzip_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + gzip.decompress(data[filename]) + except OSError: + self.fail("Received invalid gzip data.") + + @defer.inlineCallbacks + def test_gzip_plugin_compresslevel(self): + + filename_to_compressed = { + self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0), + self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('compresslevel_0'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 0, + 'gzip_mtime': 0, + 'gzip_filename': "", + }, + self._named_tempfile('compresslevel_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 9, + 'gzip_mtime': 0, + 'gzip_filename': "", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_gzip_plugin_mtime(self): + filename_to_compressed = { + self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123), + self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789), + } + + settings = { + 'FEEDS': { + self._named_tempfile('mtime_123'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 123, + 'gzip_filename': "", + }, + self._named_tempfile('mtime_123456789'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 123456789, + 'gzip_filename': "", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_gzip_plugin_filename(self): + filename_to_compressed = { + self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"), + self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"), + } + + settings = { + 'FEEDS': { + self._named_tempfile('filename_FILE1'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 0, + 'gzip_filename': "FILE1", + }, + self._named_tempfile('filename_FILE2'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 0, + 'gzip_filename': "FILE2", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin(self): + + filename = self._named_tempfile('lzma_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + lzma.decompress(data[filename]) + except lzma.LZMAError: + self.fail("Received invalid lzma data.") + + @defer.inlineCallbacks + def test_lzma_plugin_format(self): + + filename_to_compressed = { + self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ), + self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE), + } + + settings = { + 'FEEDS': { + self._named_tempfile('format_FORMAT_XZ'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_format': lzma.FORMAT_XZ, + }, + self._named_tempfile('format_FORMAT_ALONE'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_format': lzma.FORMAT_ALONE, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_check(self): + + filename_to_compressed = { + self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE), + self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256), + } + + settings = { + 'FEEDS': { + self._named_tempfile('check_CHECK_NONE'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_check': lzma.CHECK_NONE, + }, + self._named_tempfile('check_CHECK_CRC256'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_check': lzma.CHECK_SHA256, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_preset(self): + + filename_to_compressed = { + self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0), + self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('preset_PRESET_0'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_preset': 0, + }, + self._named_tempfile('preset_PRESET_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_preset': 9, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_filters(self): + import sys + if "PyPy" in sys.version: + # https://foss.heptapod.net/pypy/pypy/-/issues/3527 + raise unittest.SkipTest("lzma filters doesn't work in PyPy") + + filters = [{'id': lzma.FILTER_LZMA2}] + compressed = lzma.compress(self.expected, filters=filters) + filename = self._named_tempfile('filters') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_filters': filters, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(compressed, data[filename]) + result = lzma.decompress(data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_bz2_plugin(self): + + filename = self._named_tempfile('bz2_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + bz2.decompress(data[filename]) + except OSError: + self.fail("Received invalid bz2 data.") + + @defer.inlineCallbacks + def test_bz2_plugin_compresslevel(self): + + filename_to_compressed = { + self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1), + self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('compresslevel_1'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + 'bz2_compresslevel': 1, + }, + self._named_tempfile('compresslevel_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + 'bz2_compresslevel': 9, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = bz2.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_custom_plugin(self): + filename = self._named_tempfile('csv_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(self.expected, data[filename]) + + @defer.inlineCallbacks + def test_custom_plugin_with_parameter(self): + + expected = b'foo\r\n\nbar\r\n\n' + filename = self._named_tempfile('newline') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + 'plugin1_char': b'\n' + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(expected, data[filename]) + + @defer.inlineCallbacks + def test_custom_plugin_with_compression(self): + + expected = b'foo\r\n\nbar\r\n\n' + + filename_to_decompressor = { + self._named_tempfile('bz2'): bz2.decompress, + self._named_tempfile('lzma'): lzma.decompress, + self._named_tempfile('gzip'): gzip.decompress, + } + + settings = { + 'FEEDS': { + self._named_tempfile('bz2'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'], + 'plugin1_char': b'\n', + }, + self._named_tempfile('lzma'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'], + 'plugin1_char': b'\n', + }, + self._named_tempfile('gzip'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], + 'plugin1_char': b'\n', + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, decompressor in filename_to_decompressor.items(): + result = decompressor(data[filename]) + self.assertEqual(expected, result) + + @defer.inlineCallbacks + def test_exports_compatibility_with_postproc(self): + import marshal + import pickle + filename_to_expected = { + self._named_tempfile('csv'): b'foo\r\nbar\r\n', + self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]', + self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n', + self._named_tempfile('xml'): b'\n' + b'\nbar\n', + } + + settings = { + 'FEEDS': { + self._named_tempfile('csv'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + # empty plugin to activate postprocessing.PostProcessingManager + }, + self._named_tempfile('json'): { + 'format': 'json', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('jsonlines'): { + 'format': 'jsonlines', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('xml'): { + 'format': 'xml', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('marshal'): { + 'format': 'marshal', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('pickle'): { + 'format': 'pickle', + 'postprocessing': [self.MyPlugin1], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, result in data.items(): + if 'pickle' in filename: + expected, result = self.items[0], pickle.loads(result) + elif 'marshal' in filename: + expected, result = self.items[0], marshal.loads(result) + else: + expected = filename_to_expected[filename] + self.assertEqual(expected, result) + + class BatchDeliveriesTest(FeedExportTestBase): __test__ = True _file_mark = '_%(batch_time)s_#%(batch_id)02d_' @@ -2112,3 +2608,166 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): ), ) ) + + +class URIParamsTest: + + spider_name = "uri_params_spider" + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + raise NotImplementedError + + def test_default(self): + settings = self.build_settings( + uri='file:///tmp/%(name)s', + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_none(self): + def uri_params(params, spider): + pass + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual( + messages, + ( + ( + 'Modifying the params dictionary in-place in the ' + 'function defined in the FEED_URI_PARAMS setting or ' + 'in the uri_params key of the FEEDS setting is ' + 'deprecated. The function must return a new ' + 'dictionary instead.' + ), + ) + ) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_empty_dict(self): + def uri_params(params, spider): + return {} + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + with self.assertRaises(KeyError): + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + def test_params_as_is(self): + def uri_params(params, spider): + return params + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_custom_param(self): + def uri_params(params, spider): + return {**params, 'foo': self.spider_name} + + settings = self.build_settings( + uri='file:///tmp/%(foo)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + +class URIParamsSettingTest(URIParamsTest, unittest.TestCase): + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + extra_settings = {} + if uri_params: + extra_settings['FEED_URI_PARAMS'] = uri_params + return { + 'FEED_URI': uri, + **extra_settings, + } + + +class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + options = { + 'format': 'jl', + } + if uri_params: + options['uri_params'] = uri_params + return { + 'FEEDS': { + uri: options, + }, + } diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 677ede92b..49c83132f 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -201,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase): self.site = Site(root, timeout=None) # Start server for testing - self.hostname = u'localhost' + self.hostname = 'localhost' context_factory = ssl_context_factory(self.key_file, self.certificate_file) server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 540e27907..08420332c 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -34,7 +34,6 @@ class WrappedRequestTest(TestCase): self.assertTrue(self.wrapped.unverifiable) def test_get_origin_req_host(self): - self.assertEqual(self.wrapped.get_origin_req_host(), 'www.example.com') self.assertEqual(self.wrapped.origin_req_host, 'www.example.com') def test_has_header(self): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index b610087bd..579ef9fa2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1217,18 +1217,17 @@ class FormRequestTest(RequestTest): response, formcss="input[name='abc']") def test_from_response_valid_form_methods(self): - body = """
- -
""" + form_methods = [[method, method] for method in self.request_class.valid_form_methods] + form_methods.append(['UNKNOWN', 'GET']) - for method in self.request_class.valid_form_methods: - response = _buildresponse(body % method) + for method, expected in form_methods: + response = _buildresponse( + f'
' + '' + '
' + ) r = self.request_class.from_response(response) - self.assertEqual(r.method, method) - - response = _buildresponse(body % 'UNKNOWN') - r = self.request_class.from_response(response) - self.assertEqual(r.method, 'GET') + self.assertEqual(r.method, expected) def _buildresponse(body, **kwargs): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index c376a46cd..2986f884f 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,10 +1,8 @@ import unittest from unittest import mock -from warnings import catch_warnings, filterwarnings from w3lib.encoding import resolve_encoding -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector @@ -19,7 +17,7 @@ class BaseResponseTest(unittest.TestCase): response_class = Response def test_init(self): - # Response requires url in the consturctor + # Response requires url in the constructor self.assertRaises(Exception, self.response_class) self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class)) self.assertRaises(TypeError, self.response_class, b"http://example.com") @@ -134,9 +132,6 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertEqual(response.body_as_unicode(), body_unicode) self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): @@ -346,12 +341,6 @@ class TextResponseTest(BaseResponseTest): original_string = unicode_string.encode('cp1251') r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') - # check body_as_unicode - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertTrue(isinstance(r1.body_as_unicode(), str)) - self.assertEqual(r1.body_as_unicode(), unicode_string) - # check response.text self.assertTrue(isinstance(r1.text, str)) self.assertEqual(r1.text, unicode_string) @@ -392,7 +381,7 @@ class TextResponseTest(BaseResponseTest): def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" r = self.response_class("http://www.example.com", - headers={"Content-type": ["text/html; charset=UKNOWN"]}, + headers={"Content-type": ["text/html; charset=UNKNOWN"]}, body=b"\xc2\xa3") self.assertEqual(r._declared_encoding(), None) self._assert_response_values(r, 'utf-8', "\xa3") @@ -683,13 +672,6 @@ class TextResponseTest(BaseResponseTest): with self.assertRaises(ValueError): response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]') - def test_body_as_unicode_deprecation_warning(self): - with catch_warnings(record=True) as warnings: - r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8') - self.assertEqual(r1.body_as_unicode(), 'Hello') - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - def test_json_response(self): json_body = b"""{"ip": "109.187.217.200"}""" json_response = self.response_class("http://www.example.com", body=json_body) diff --git a/tests/test_item.py b/tests/test_item.py index c94bb44af..25f2aea0a 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,9 +1,7 @@ import unittest from unittest import mock -from warnings import catch_warnings, filterwarnings -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta +from scrapy.item import ABCMeta, Field, Item, ItemMeta class ItemTest(unittest.TestCase): @@ -254,18 +252,6 @@ class ItemTest(unittest.TestCase): item['tags'].append('tag2') assert item['tags'] != copied_item['tags'] - def test_dictitem_deprecation_warning(self): - """Make sure the DictItem deprecation warning is not issued for - Item""" - with catch_warnings(record=True) as warnings: - Item() - self.assertEqual(len(warnings), 0) - - class SubclassedItem(Item): - pass - SubclassedItem() - self.assertEqual(len(warnings), 0) - class ItemMetaTest(unittest.TestCase): @@ -303,94 +289,5 @@ class ItemMetaClassCellRegression(unittest.TestCase): super().__init__(*args, **kwargs) -class DictItemTest(unittest.TestCase): - - def test_deprecation_warning(self): - with catch_warnings(record=True) as warnings: - DictItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: - class SubclassedDictItem(DictItem): - pass - SubclassedDictItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - -class BaseItemTest(unittest.TestCase): - - def test_isinstance_check(self): - - class SubclassedBaseItem(BaseItem): - pass - - class SubclassedItem(Item): - pass - - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) - self.assertTrue(isinstance(Item(), BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), BaseItem)) - - # make sure internal checks using private _BaseItem class succeed - self.assertTrue(isinstance(BaseItem(), _BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), _BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) - - def test_deprecation_warning(self): - """ - Make sure deprecation warnings are logged whenever BaseItem is used, - either instantiated or in an isinstance check - """ - with catch_warnings(record=True) as warnings: - BaseItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - - class SubclassedBaseItem(BaseItem): - pass - - SubclassedBaseItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - self.assertFalse(isinstance("foo", BaseItem)) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - -class ItemNoDeprecationWarningTest(unittest.TestCase): - def test_no_deprecation_warning(self): - """ - Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used. - """ - class SubclassedItem(Item): - pass - - with catch_warnings(record=True) as warnings: - Item() - SubclassedItem() - _BaseItem() - self.assertFalse(isinstance("foo", _BaseItem)) - self.assertFalse(isinstance("foo", Item)) - self.assertFalse(isinstance("foo", SubclassedItem)) - self.assertTrue(isinstance(_BaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), Item)) - self.assertTrue(isinstance(SubclassedItem(), SubclassedItem)) - self.assertEqual(len(warnings), 0) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_loader.py b/tests/test_loader.py index b0bc82f4e..f7ab1f236 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -4,7 +4,7 @@ import attr from itemadapter import ItemAdapter from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, Response from scrapy.item import Item, Field from scrapy.loader import ItemLoader from scrapy.selector import Selector @@ -304,6 +304,12 @@ class SelectortemLoaderTest(unittest.TestCase): l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), ['Marta']) + + def test_init_method_with_base_response(self): + """Selector should be None after initialization""" + response = Response("https://scrapy.org") + l = TestItemLoader(response=response) + self.assertIs(l.selector, None) def test_init_method_with_response(self): l = TestItemLoader(response=self.response) diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 41afa2896..0fd52da5f 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -703,7 +703,7 @@ class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): return None with warnings.catch_warnings(record=True) as w: - wrap_loader_context(function, context=dict()) + wrap_loader_context(function, context={}) assert len(w) == 1 assert issubclass(w[0].category, ScrapyDeprecationWarning) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index ff3af9a74..8e432b913 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -6,6 +6,7 @@ from twisted.internet.defer import Deferred from twisted.trial import unittest from scrapy import Spider, signals, Request +from scrapy.utils.defer import maybe_deferred_to_future, deferred_to_future from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver import MockServer @@ -31,18 +32,38 @@ class DeferredPipeline: class AsyncDefPipeline: async def process_item(self, item, spider): - await defer.succeed(42) + d = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, None) + await maybe_deferred_to_future(d) item['pipeline_passed'] = True return item class AsyncDefAsyncioPipeline: async def process_item(self, item, spider): + d = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, None) + await deferred_to_future(d) await asyncio.sleep(0.2) item['pipeline_passed'] = await get_from_asyncio_queue(True) return item +class AsyncDefNotAsyncioPipeline: + async def process_item(self, item, spider): + d1 = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d1.callback, None) + await d1 + d2 = Deferred() + reactor.callLater(0, d2.callback, None) + await maybe_deferred_to_future(d2) + item['pipeline_passed'] = True + return item + + class ItemSpider(Spider): name = 'itemspider' @@ -99,3 +120,10 @@ class PipelineTestCase(unittest.TestCase): crawler = self._create_crawler(AsyncDefAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 1) + + @mark.only_not_asyncio() + @defer.inlineCallbacks + def test_asyncdef_not_asyncio_pipeline(self): + crawler = self._create_crawler(AsyncDefNotAsyncioPipeline) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(len(self.items), 1) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 00c532c41..25d9657d5 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -106,9 +106,9 @@ class CrawlTestCase(TestCase): """ Downloader middleware which returns a response with an specific 'request' attribute. - * The spider callback should receive the overriden response.request - * Handlers listening to the response_received signal should receive the overriden response.request - * The "crawled" log message should show the overriden response.request + * The spider callback should receive the overridden response.request + * Handlers listening to the response_received signal should receive the overridden response.request + * The "crawled" log message should show the overridden response.request """ signal_params = {} @@ -144,7 +144,7 @@ class CrawlTestCase(TestCase): An exception is raised but caught by the next middleware, which returns a Response with a specific 'request' attribute. - The spider callback should receive the overriden response.request + The spider callback should receive the overridden response.request """ url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index b68184b87..473a93e69 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -57,10 +57,10 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks = list() + checks = [] def start_requests(self): - data = {'key': 'value', 'number': 123} + data = {'key': 'value', 'number': 123, 'callback': 'some_callback'} yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data) yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data) yield Request(self.mockserver.url('/general_without'), self.parse_general) @@ -88,7 +88,8 @@ class KeywordArgumentsSpider(MockServerSpider): if response.url.endswith('/general_with'): self.checks.append(kwargs['key'] == 'value') self.checks.append(kwargs['number'] == 123) - self.crawler.stats.inc_value('boolean_checks', 2) + self.checks.append(kwargs['callback'] == 'some_callback') + self.crawler.stats.inc_value('boolean_checks', 3) elif response.url.endswith('/general_without'): self.checks.append(kwargs == {}) self.crawler.stats.inc_value('boolean_checks') @@ -104,13 +105,13 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(default == 99) self.crawler.stats.inc_value('boolean_checks', 4) - def parse_takes_less(self, response, key): + def parse_takes_less(self, response, key, callback): """ Should raise TypeError: parse_takes_less() got an unexpected keyword argument 'number' """ - def parse_takes_more(self, response, key, number, other): + def parse_takes_more(self, response, key, number, callback, other): """ Should raise TypeError: parse_takes_more() missing 1 required positional argument: 'other' @@ -161,11 +162,13 @@ class CallbackKeywordArgumentsTestCase(TestCase): self.assertTrue( str(exceptions['takes_less'].exc_info[1]).endswith( "parse_takes_less() got an unexpected keyword argument 'number'" - ) + ), + msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]), ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) self.assertTrue( str(exceptions['takes_more'].exc_info[1]).endswith( "parse_takes_more() missing 1 required positional argument: 'other'" - ) + ), + msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]), ) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 373b2e49c..4d4483881 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -22,7 +22,7 @@ class SignalCatcherSpider(Spider): return spider def on_request_left(self, request, spider): - self.caught_times = self.caught_times + 1 + self.caught_times += 1 class TestCatching(TestCase): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 512a7460e..2d4bfa165 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -22,7 +22,7 @@ MockSlot = collections.namedtuple('MockSlot', ['active']) class MockDownloader: def __init__(self): - self.slots = dict() + self.slots = {} def _get_slot_key(self, request, spider): if Downloader.DOWNLOAD_SLOT in request.meta: @@ -31,7 +31,7 @@ class MockDownloader: return urlparse_cached(request).hostname or '' def increment(self, slot_key): - slot = self.slots.setdefault(slot_key, MockSlot(active=list())) + slot = self.slots.setdefault(slot_key, MockSlot(active=[])) slot.active.append(1) def decrement(self, slot_key): @@ -114,7 +114,7 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): for url, priority in _PRIORITIES: self.scheduler.enqueue_request(Request(url, priority=priority)) - priorities = list() + priorities = [] while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) @@ -167,7 +167,7 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): self.close_scheduler() self.create_scheduler() - priorities = list() + priorities = [] while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) @@ -259,7 +259,7 @@ class DownloaderAwareSchedulerTestMixin: self.close_scheduler() self.create_scheduler() - dequeued_slots = list() + dequeued_slots = [] requests = [] downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): diff --git a/tests/test_spider.py b/tests/test_spider.py index d23543f6a..689349999 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -21,6 +21,7 @@ from scrapy.spiders import ( ) from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler +from tests import get_testdata class SpiderTest(unittest.TestCase): @@ -167,6 +168,23 @@ class CSVFeedSpiderTest(SpiderTest): spider_class = CSVFeedSpider + def test_parse_rows(self): + body = get_testdata('feeds', 'feed-sample6.csv') + response = Response("http://example.org/dummy.csv", body=body) + + class _CrawlSpider(self.spider_class): + name = "test" + delimiter = "," + quotechar = "'" + + def parse_row(self, response, row): + return row + + spider = _CrawlSpider() + rows = list(spider.parse_rows(response)) + assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'} + assert len(rows) == 4 + class CrawlSpiderTest(SpiderTest): @@ -584,39 +602,6 @@ class DeprecationTest(unittest.TestCase): assert issubclass(CrawlSpider, Spider) assert isinstance(CrawlSpider(name='foo'), Spider) - def test_make_requests_from_url_deprecated(self): - class MySpider4(Spider): - name = 'spider1' - start_urls = ['http://example.com'] - - class MySpider5(Spider): - name = 'spider2' - start_urls = ['http://example.com'] - - def make_requests_from_url(self, url): - return Request(url + "/foo", dont_filter=True) - - with warnings.catch_warnings(record=True) as w: - # spider without overridden make_requests_from_url method - # doesn't issue a warning - spider1 = MySpider4() - self.assertEqual(len(list(spider1.start_requests())), 1) - self.assertEqual(len(w), 0) - - # spider without overridden make_requests_from_url method - # should issue a warning when called directly - request = spider1.make_requests_from_url("http://www.example.com") - self.assertTrue(isinstance(request, Request)) - self.assertEqual(len(w), 1) - - # spider with overridden make_requests_from_url issues a warning, - # but the method still works - spider2 = MySpider5() - requests = list(spider2.start_requests()) - self.assertEqual(len(requests), 1) - self.assertEqual(requests[0].url, 'http://example.com/foo') - self.assertEqual(len(w), 2) - class NoParseMethodSpiderTest(unittest.TestCase): diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a2114bd18..295323e4d 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,6 +1,4 @@ -import platform -import sys -from unittest import skipIf, TestCase +from unittest import TestCase from pytest import mark @@ -14,9 +12,6 @@ class AsyncioTest(TestCase): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index dc2560add..a92880626 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -176,7 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": (1, 2, 3, 4), "batch_item_count": 2, - "item_export_kwargs": dict(), + "item_export_kwargs": {}, }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -199,7 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": None, "batch_item_count": 2, - "item_export_kwargs": dict(), + "item_export_kwargs": {}, }) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 7a5f458c7..032dbc8c5 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -23,7 +23,7 @@ class MustbeDeferredTest(unittest.TestCase): dfd = mustbe_deferred(_append, 1) dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + steps.append(2) # add another value, that should be caught by assertEqual return dfd def test_unfired_deferred(self): @@ -37,7 +37,7 @@ class MustbeDeferredTest(unittest.TestCase): dfd = mustbe_deferred(_append, 1) dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + steps.append(2) # add another value, that should be caught by assertEqual return dfd diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 47d73a2dd..b83c1d6f0 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -27,7 +27,7 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') - self.assertRaises(TypeError, load_object, dict()) + self.assertRaises(TypeError, load_object, {}) def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d6f4c0bb5..0a09f6109 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(response_status_message(200), '200 OK') self.assertEqual(response_status_message(404), '404 Not Found') self.assertEqual(response_status_message(573), "573 Unknown Status") + + def test_inject_base_url(self): + url = "http://www.example.com" + + def check_base_url(burl): + path = urlparse(burl).path + if not os.path.exists(path): + path = burl.replace('file://', '') + with open(path, "rb") as f: + bbody = f.read() + self.assertEqual(bbody.count(b''), 1) + return True + + r1 = HtmlResponse(url, body=b""" + + Dummy +

Hello world.

+ """) + r2 = HtmlResponse(url, body=b""" + + Dummy + Hello world. + """) + r3 = HtmlResponse(url, body=b""" + + Dummy + +
Hello header
+

Hello world.

+ + """) + r4 = HtmlResponse(url, body=b""" + + + Dummy +

Hello world.

+ """) + r5 = HtmlResponse(url, body=b""" + + + + Standard head + +

Hello world.

+ """) + + assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" + assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head" + assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag" + assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment" + assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment" diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5ff2e41ef..1d5e63363 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -36,7 +36,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): self.assertEqual(result.read().decode('utf8'), rendered) os.remove(render_path) - assert not os.path.exists(render_path) # Failure of test iself + assert not os.path.exists(render_path) # Failure of test itself if '__main__' == __name__: diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 6e4cb9b6e..a6d55cb38 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -21,14 +21,6 @@ except ImportError: from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks -from twisted.web.test.test_webclient import ( - ForeverTakingResource, - ErrorResource, - NoLengthResource, - HostHeaderResource, - PayloadResource, - BrokenDownloadResource, -) from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory @@ -36,7 +28,15 @@ from scrapy.http import Request, Headers from scrapy.settings import Settings from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes, to_unicode -from tests.mockserver import ssl_context_factory +from tests.mockserver import ( + BrokenDownloadResource, + ErrorResource, + ForeverTakingResource, + HostHeaderResource, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): diff --git a/tox.ini b/tox.ini index e274fc8d2..db151f215 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,8 @@ deps = #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' + # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) + markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' # Extras botocore>=1.4.87 passenv = @@ -29,7 +31,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - py.test --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} install_command = pip install -U -ctests/upper-constraints.txt {opts} {packages} @@ -41,12 +43,13 @@ deps = types-pyOpenSSL==20.0.3 types-setuptools==57.0.0 commands = + pip install types-dataclasses # remove once py36 support is dropped mypy --show-error-codes {posargs: scrapy tests} [testenv:security] basepython = python3 deps = - bandit + bandit==1.7.3 commands = bandit -r -c .bandit.yml {posargs:scrapy} @@ -57,14 +60,15 @@ deps = # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 pytest-flake8 + flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 commands = - py.test --flake8 {posargs:docs scrapy tests} + pytest --flake8 {posargs:docs scrapy tests} [testenv:pylint] basepython = python3 deps = {[testenv:extra-deps]deps} - pylint + pylint==2.12.2 commands = pylint conftest.py docs extras scrapy setup.py tests @@ -125,6 +129,9 @@ deps = robotexclusionrulesparser Pillow>=4.0.0 Twisted[http2]>=17.9.0 + # Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps, + # so we need to pin old markupsafe here too + markupsafe < 2.1.0 [testenv:asyncio] commands = @@ -140,7 +147,7 @@ setenv = [testenv:pypy3] basepython = pypy3 commands = - py.test {posargs:--durations=10 docs scrapy tests} + pytest {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython}