diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c9f1abea5..d9e4a2831 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,8 +1,7 @@ [bumpversion] -current_version = 1.8.0 +current_version = 2.5.0 commit = True tag = True tag_name = {new_version} [bumpversion:file:scrapy/VERSION] - diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..1c503fb0b --- /dev/null +++ b/.flake8 @@ -0,0 +1,19 @@ +[flake8] + +max-line-length = 119 +ignore = W503 + +exclude = +# Exclude files that are meant to provide top-level imports +# E402: Module level import not at top of file +# F401: Module imported but unused + scrapy/__init__.py E402 + scrapy/core/downloader/handlers/http.py F401 + scrapy/http/__init__.py F401 + scrapy/linkextractors/__init__.py E402 F401 + scrapy/selector/__init__.py F401 + scrapy/spiders/__init__.py E402 F401 + + # Issues pending a review: + scrapy/utils/url.py F403 F405 + tests/test_loader.py E741 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfbdf4208 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/sample_data/** binary diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 000000000..80df9469d --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,45 @@ +name: Checks +on: [push, pull_request] + +jobs: + checks: + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + include: + - python-version: "3.10" + env: + TOXENV: security + - python-version: "3.10" + env: + TOXENV: flake8 + # Pylint requires installing reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 + - python-version: 3.8 + env: + TOXENV: pylint + TOX_PIP_VERSION: 20.3.3 + - python-version: 3.6 + env: + TOXENV: typing + - python-version: "3.10" # Keep in sync with .readthedocs.yml + env: + TOXENV: docs + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run check + env: ${{ matrix.env }} + run: | + if [[ ! -z "$TOX_PIP_VERSION" ]]; then + pip install tox-pip-version + fi + pip install -U tox + tox diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..44b682830 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,31 @@ +name: Publish +on: [push] + +jobs: + publish: + runs-on: ubuntu-18.04 + if: startsWith(github.event.ref, 'refs/tags/') + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Check Tag + id: check-release-tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then + echo ::set-output name=release_tag::true + fi + + - name: Publish to PyPI + if: steps.check-release-tag.outputs.release_tag == 'true' + run: | + pip install --upgrade setuptools wheel twine + python setup.py sdist bdist_wheel + export TWINE_USERNAME=__token__ + export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} + twine upload dist/* diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml new file mode 100644 index 000000000..3aaf688c7 --- /dev/null +++ b/.github/workflows/tests-macos.yml @@ -0,0 +1,26 @@ +name: macOS +on: [push, pull_request] + +jobs: + tests: + runs-on: macos-10.15 + strategy: + fail-fast: false + matrix: + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + run: | + pip install -U tox + tox -e py + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml new file mode 100644 index 000000000..5ea50e644 --- /dev/null +++ b/.github/workflows/tests-ubuntu.yml @@ -0,0 +1,83 @@ +name: Ubuntu +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + include: + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + - python-version: 3.9 + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio + - python-version: pypy3 + env: + TOXENV: pypy3 + PYPY_VERSION: 3.6-v7.3.3 + + # pinned deps + - python-version: 3.6.12 + env: + TOXENV: pinned + - python-version: 3.6.12 + env: + TOXENV: asyncio-pinned + - python-version: pypy3 + env: + TOXENV: pypy3-pinned + PYPY_VERSION: 3.6-v7.2.0 + + # extras + # extra-deps includes reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 + - python-version: 3.8 + env: + TOXENV: extra-deps + TOX_PIP_VERSION: 20.3.3 + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system libraries + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4' + run: | + sudo apt-get update + # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version + sudo apt-get install libxml2-dev/bionic-updates libxslt-dev + + - name: Run tests + env: ${{ matrix.env }} + run: | + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + if [[ ! -z "$TOX_PIP_VERSION" ]]; then + pip install tox-pip-version + fi + pip install -U tox + tox + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml new file mode 100644 index 000000000..ab7385118 --- /dev/null +++ b/.github/workflows/tests-windows.yml @@ -0,0 +1,42 @@ +name: Windows +on: [push, pull_request] + +jobs: + tests: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - python-version: 3.6 + env: + TOXENV: windows-pinned + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + - python-version: 3.9 + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.gitignore b/.gitignore index ff6e2ea65..d77d24624 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,12 @@ htmlcov/ .coverage .pytest_cache/ .coverage.* +coverage.* +test-output.* .cache/ +.mypy_cache/ +/tests/keys/localhost.crt +/tests/keys/localhost.key # Windows Thumbs.db diff --git a/.readthedocs.yml b/.readthedocs.yml index 17eba34f3..390be3749 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,11 +1,17 @@ version: 2 +formats: all sphinx: configuration: docs/conf.py fail_on_warning: true + +build: + os: ubuntu-20.04 + tools: + # For available versions, see: + # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python + python: "3.10" # Keep in sync with .github/workflows/checks.yml + python: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image - version: 3.7 # Keep in sync with .travis.yml install: - requirements: docs/requirements.txt - path: . diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 66e1a9617..000000000 --- a/.travis.yml +++ /dev/null @@ -1,65 +0,0 @@ -language: python -dist: xenial -branches: - only: - - master - - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ -matrix: - include: - - env: TOXENV=security - python: 3.8 - - env: TOXENV=flake8 - python: 3.8 - - env: TOXENV=pypy3 - - env: TOXENV=py35 - python: 3.5 - - env: TOXENV=pinned - python: 3.5 - - env: TOXENV=py35-asyncio - python: 3.5.2 - - env: TOXENV=py36 - python: 3.6 - - env: TOXENV=py37 - python: 3.7 - - env: TOXENV=py38 - python: 3.8 - - env: TOXENV=extra-deps - python: 3.8 - - env: TOXENV=py38-asyncio - python: 3.8 - - env: TOXENV=docs - python: 3.7 # Keep in sync with .readthedocs.yml -install: - - | - if [ "$TOXENV" = "pypy3" ]; then - export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable" - wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - - pip install -U tox twine wheel codecov - -script: tox -after_success: - - codecov -notifications: - irc: - use_notice: true - skip_join: true - channels: - - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: scrapy - password: - secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA= - on: - tags: true - repo: scrapy/scrapy - condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" diff --git a/AUTHORS b/AUTHORS index bcaa1ecd3..9706adf42 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,8 +1,8 @@ Scrapy was brought to life by Shane Evans while hacking a scraping framework prototype for Mydeco (mydeco.com). It soon became maintained, extended and improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to -bootstrap the project. In mid-2011, Scrapinghub became the new official -maintainer. +bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new +official maintainer. Here is the list of the primary authors & contributors: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d1cd3e517..902cd523e 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -55,7 +55,7 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@zyte.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. @@ -72,3 +72,6 @@ available at [http://contributor-covenant.org/version/1/4][version]. [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/Makefile.buildbot b/Makefile.buildbot deleted file mode 100644 index 775538259..000000000 --- a/Makefile.buildbot +++ /dev/null @@ -1,24 +0,0 @@ -TRIAL := $(shell which trial) -BRANCH := $(shell git rev-parse --abbrev-ref HEAD) -export PYTHONPATH=$(PWD) - -test: - coverage run --branch $(TRIAL) --reporter=text tests - rm -rf htmlcov && coverage html - -s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/ - -build: - git describe --tags --match '[0-9]*' |sed 's/-/.post/;s/-g/+g/' >scrapy/VERSION - debchange -m -D unstable --force-distribution -v \ - $$(python setup.py --version |sed -r 's/([0-9]+.[0-9]+.[0-9]+)(a|b|rc|dev)([0-9]*)/\1~\2\3/')-$$(date +%s) \ - "Automatic build" - debuild -us -uc -b - -clean: - git checkout debian scrapy/VERSION - git clean -dfq - -pypi: - umask 0022 && chmod -R a+rX . && python setup.py sdist upload - -.PHONY: clean test build diff --git a/README.rst b/README.rst index ce5973bcd..05f10bb6c 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,6 @@ +.. image:: /artwork/scrapy-logo.jpg + :width: 400px + ====== Scrapy ====== @@ -10,9 +13,17 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg - :target: https://travis-ci.org/scrapy/scrapy - :alt: Build Status +.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu + :alt: Ubuntu + +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS + +.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows + :alt: Windows .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy @@ -34,13 +45,20 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other +contributors`_. + +.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Zyte: https://www.zyte.com/ + Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. + Requirements ============ -* Python 3.5+ +* Python 3.6+ * Works on Linux, Windows, macOS, BSD Install @@ -81,7 +99,7 @@ Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@scrapinghub.com. +Please report unacceptable behavior to opensource@zyte.com. Companies using Scrapy ====================== diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 7fd636864..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,25 +0,0 @@ -platform: x86 -version: '{branch}-{build}' -environment: - matrix: - - PYTHON: "C:\\Python36" - TOX_ENV: py36 - -branches: - only: - - master - - /d+\.\d+\.\d+[\w\-]*$/ - -install: - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%" - - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE" - - "pip install -U tox" - -build: false -skip_tags: true -test_script: - - "tox -e %TOX_ENV%" - -cache: - - '%LOCALAPPDATA%\pip\cache' diff --git a/conftest.py b/conftest.py index be5fbabf4..117087790 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,11 @@ from pathlib import Path import pytest +from twisted.web.http import H2_ENABLED + +from scrapy.utils.reactor import install_reactor + +from tests.keys import generate_keys def _py_files(folder): @@ -12,14 +17,23 @@ collect_ignore = [ "scrapy/utils/testsite.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), - # Py36-only parts of respective tests - *_py_files("tests/py36"), + # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess + *_py_files("tests/CrawlerRunner"), ] -for line in open('tests/ignores.txt'): - file_path = line.strip() - if file_path and file_path[0] != '#': - collect_ignore.append(file_path) +with open('tests/ignores.txt') as reader: + for line in reader: + file_path = line.strip() + if file_path and file_path[0] != '#': + collect_ignore.append(file_path) + +if not H2_ENABLED: + collect_ignore.extend( + ( + 'scrapy/core/downloader/handlers/http2.py', + *_py_files("scrapy/core/http2"), + ) + ) @pytest.fixture() @@ -38,6 +52,14 @@ def pytest_collection_modifyitems(session, config, items): pass +def pytest_addoption(parser): + parser.addoption( + "--reactor", + default="default", + choices=["default", "asyncio"], + ) + + @pytest.fixture(scope='class') def reactor_pytest(request): if not request.cls: @@ -51,3 +73,18 @@ def reactor_pytest(request): def only_asyncio(request, reactor_pytest): if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio': pytest.skip('This test is only run with --reactor=asyncio') + + +@pytest.fixture(autouse=True) +def only_not_asyncio(request, reactor_pytest): + if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio': + pytest.skip('This test is only run without --reactor=asyncio') + + +def pytest_configure(config): + if config.getoption("--reactor") == "asyncio": + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + +# Generate localhost certificate files, needed by some tests +generate_keys() diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index dde97f9e3..000000000 --- a/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -scrapy (0.11) unstable; urgency=low - - * Initial release. - - -- Scrapinghub Team Thu, 10 Jun 2010 17:24:02 -0300 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index 7f8f011eb..000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -7 diff --git a/debian/control b/debian/control deleted file mode 100644 index 2cc8eedf4..000000000 --- a/debian/control +++ /dev/null @@ -1,20 +0,0 @@ -Source: scrapy -Section: python -Priority: optional -Maintainer: Scrapinghub Team -Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2) -Standards-Version: 3.8.4 -Homepage: https://scrapy.org/ - -Package: scrapy -Architecture: all -Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl, - python-w3lib (>= 1.8.0), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2) -Recommends: python-setuptools -Conflicts: python-scrapy, scrapy-0.25 -Provides: python-scrapy, scrapy-0.25 -Description: Python web crawling and web scraping framework - Scrapy is a fast high-level web crawling and web scraping framework, - used to crawl websites and extract structured data from their pages. - It can be used for a wide range of purposes, from data mining to - monitoring and automated testing. diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index c1bf47565..000000000 --- a/debian/copyright +++ /dev/null @@ -1,40 +0,0 @@ -This package was debianized by the Scrapinghub team . - -It was downloaded from https://scrapy.org - -Upstream Author: Scrapy Developers - -Copyright: 2007-2013 Scrapy Developers - -License: bsd - -Copyright (c) Scrapy developers. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of Scrapy nor the names of its contributors may be used - to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The Debian packaging is (C) 2010-2013, Scrapinghub and -is licensed under the BSD, see `/usr/share/common-licenses/BSD'. diff --git a/debian/pyversions b/debian/pyversions deleted file mode 100644 index 1effb0034..000000000 --- a/debian/pyversions +++ /dev/null @@ -1 +0,0 @@ -2.7 diff --git a/debian/rules b/debian/rules deleted file mode 100755 index b8796e6e3..000000000 --- a/debian/rules +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/make -f -# -*- makefile -*- - -%: - dh $@ diff --git a/debian/scrapy.docs b/debian/scrapy.docs deleted file mode 100644 index c19ffba4d..000000000 --- a/debian/scrapy.docs +++ /dev/null @@ -1,2 +0,0 @@ -README.rst -AUTHORS diff --git a/debian/scrapy.install b/debian/scrapy.install deleted file mode 100644 index c288ebed3..000000000 --- a/debian/scrapy.install +++ /dev/null @@ -1,2 +0,0 @@ -extras/scrapy_bash_completion etc/bash_completion.d/ -extras/scrapy_zsh_completion /usr/share/zsh/vendor-completions/_scrapy diff --git a/debian/scrapy.lintian-overrides b/debian/scrapy.lintian-overrides deleted file mode 100644 index b5de7f67d..000000000 --- a/debian/scrapy.lintian-overrides +++ /dev/null @@ -1 +0,0 @@ -new-package-should-close-itp-bug diff --git a/debian/scrapy.manpages b/debian/scrapy.manpages deleted file mode 100644 index 4818e9c92..000000000 --- a/debian/scrapy.manpages +++ /dev/null @@ -1 +0,0 @@ -extras/scrapy.1 diff --git a/docs/Makefile b/docs/Makefile index ff68bf1ae..87d5d3047 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,7 +8,7 @@ PYTHON = python SPHINXOPTS = PAPER = SOURCES = -SHELL = /bin/bash +SHELL = /usr/bin/env bash ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ -D latex_elements.papersize=$(PAPER) \ diff --git a/docs/README.rst b/docs/README.rst index 0a343cd19..0b7afa548 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -57,3 +57,12 @@ There is a way to recreate the doc automatically when you make changes, you need to install watchdog (``pip install watchdog``) and then use:: make watch + +Alternative method using tox +---------------------------- + +To compile the documentation to HTML run the following command:: + + tox -e docs + +Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir. diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 192123473..640660943 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -17,7 +17,7 @@ class SettingsListDirective(Directive): def is_setting_index(node): if node.tagname == 'index': # index entries for setting directives look like: - # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')] + # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] entry_type, info, refid = node['entries'][0][:3] return entry_type == 'pair' and info.endswith('; setting') return False diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 000000000..64f16939c --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,10 @@ +/* Move lists closer to their introducing paragraph */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} +.rst-content p + ol, .rst-content p + ul { + margin-top: -18px; /* Compensates margin-top: 24px of p */ +} +.rst-content dl p + ol, .rst-content dl p + ul { + margin-top: -6px; /* Compensates margin-top: 12px of p */ +} \ No newline at end of file diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index a6f6cbda8..18a5231ee 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -3,14 +3,9 @@ {% block footer %} {{ super() }} {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index 6e2399f66..406c4d94a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Scrapy documentation build configuration file, created by # sphinx-quickstart on Mon Nov 24 12:02:52 2008. # @@ -51,7 +49,7 @@ master_doc = 'index' # General information about the project. project = 'Scrapy' -copyright = '2008–{}, Scrapy developers'.format(datetime.now().year) +copyright = f'2008–{datetime.now().year}, Scrapy developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -102,6 +100,9 @@ exclude_trees = ['.build'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' +# List of Sphinx warnings that will not be raised +suppress_warnings = ['epub.unknown_project_files'] + # Options for HTML output # ----------------------- @@ -121,7 +122,6 @@ html_theme = 'sphinx_rtd_theme' import sphinx_rtd_theme html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # The style sheet to use for HTML and HTML Help pages. A file of that name # must exist either in Sphinx' static/ path, or in one of the custom paths # given in html_static_path. @@ -182,6 +182,10 @@ html_copy_source = True # Output file base name for HTML help builder. htmlhelp_basename = 'Scrapydoc' +html_css_files = [ + 'custom.css', +] + # Options for LaTeX output # ------------------------ @@ -280,8 +284,11 @@ coverage_ignore_pyobjects = [ # ------------------------------------- intersphinx_mapping = { + 'attrs': ('https://www.attrs.org/en/stable/', None), 'coverage': ('https://coverage.readthedocs.io/en/stable', None), + 'cryptography' : ('https://cryptography.io/en/latest/', None), 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), + 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), @@ -295,3 +302,23 @@ intersphinx_mapping = { # ------------------------------------ hoverxref_auto_ref = True +hoverxref_role_types = { + "class": "tooltip", + "confval": "tooltip", + "hoverxref": "tooltip", + "mod": "tooltip", + "ref": "tooltip", +} +hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal'] + + +def setup(app): + app.connect('autodoc-skip-member', maybe_skip_member) + + +def maybe_skip_member(app, what, name, obj, skip, options): + if not skip: + # autodocs was generating a text "alias of" for the following members + # https://github.com/sphinx-doc/sphinx/issues/4422 + return name in {'default_item_class', 'default_selector_class'} + return skip diff --git a/docs/conftest.py b/docs/conftest.py index 8c735e838..a0636f8ac 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE from scrapy.http.response.html import HtmlResponse from sybil import Sybil -from sybil.parsers.codeblock import CodeBlockParser +try: + # >2.0.1 + from sybil.parsers.codeblock import PythonCodeBlockParser +except ImportError: + from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser from sybil.parsers.doctest import DocTestParser from sybil.parsers.skip import skip @@ -21,7 +25,7 @@ def setup(namespace): pytest_collect_file = Sybil( parsers=[ DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE), - CodeBlockParser(future_imports=['print_function']), + PythonCodeBlockParser(future_imports=['print_function']), skip, ], pattern='*.rst', diff --git a/docs/contributing.rst b/docs/contributing.rst index aed5ab92e..4d2580a6c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -108,6 +108,11 @@ Well-written patches should: tox -e docs-coverage +* if you are removing deprecated code, first make sure that at least 1 year + (12 months) has passed since the release that introduced the deprecation. + See :ref:`deprecation-policy`. + + .. _submitting-patches: Submitting patches @@ -135,7 +140,7 @@ original pull request author hasn't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is -not considered rude as soon as the original author is acknowledged by keeping +not considered rude as long as the original author is acknowledged by keeping his/her commits. You can pull an existing pull request to a local branch @@ -155,6 +160,9 @@ Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports removal, etc) in separate commits from functional changes. This will make pull requests easier to review and more likely to get merged. + +.. _coding-style: + Coding style ============ @@ -163,7 +171,7 @@ Scrapy: * Unless otherwise specified, follow :pep:`8`. -* It's OK to use lines longer than 80 chars if it improves the code +* It's OK to use lines longer than 79 chars if it improves the code readability. * Don't put your name in the code you contribute; git provides enough @@ -191,6 +199,17 @@ In any case, if something is covered in a docstring, use the documentation instead of duplicating the docstring in files within the ``docs/`` directory. +Documentation updates that cover new or modified features must use Sphinx’s +:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use +``VERSION`` as version, we will replace it with the actual version right before +the corresponding release. When we release a new major or minor version of +Scrapy, we remove these directives if they are older than 3 years. + +Documentation about deprecated features must be removed as those features are +deprecated, so that new readers do not run into it. New deprecations and +deprecation removals are documented in the :ref:`release notes `. + + Tests ===== diff --git a/docs/faq.rst b/docs/faq.rst index 75a0f4864..8283cab11 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars .. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use -.. _faq-python-versions: - -What Python versions does Scrapy support? ------------------------------------------ - -Scrapy is supported under Python 3.5+ -under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). -Python 3 support was added in Scrapy 1.1. -PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5. -Python 2 support was dropped in Scrapy 2.0. - -.. note:: - For Python 3 support on Windows, it is recommended to use - Anaconda/Miniconda as :ref:`outlined in the installation guide `. Did Scrapy "steal" X from Django? --------------------------------- @@ -159,6 +145,41 @@ How can I make Scrapy consume less memory? See previous question. +How can I prevent memory errors due to many allowed domains? +------------------------------------------------------------ + +If you have a spider with a long list of +:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider +replacing the default +:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware +with a :ref:`custom spider middleware ` that requires +less memory. For example: + +- If your domain names are similar enough, use your own regular expression + instead joining the strings in + :attr:`~scrapy.Spider.allowed_domains` into a complex regular + expression. + +- If you can `meet the installation requirements`_, use pyre2_ instead of + Python’s re_ to compile your URL-filtering regular expression. See + :issue:`1908`. + +See also other suggestions at `StackOverflow`_. + +.. note:: Remember to disable + :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable + your custom implementation:: + + SPIDER_MIDDLEWARES = { + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, + 'myproject.middlewares.CustomOffsiteMiddleware': 500, + } + +.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation +.. _pyre2: https://github.com/andreasvc/pyre2 +.. _re: https://docs.python.org/library/re.html +.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 + Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- @@ -250,15 +271,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file? To dump into a JSON file:: - scrapy crawl myspider -o items.json + scrapy crawl myspider -O items.json To dump into a CSV file:: - scrapy crawl myspider -o items.csv + scrapy crawl myspider -O items.csv To dump into a XML file:: - scrapy crawl myspider -o items.xml + scrapy crawl myspider -O items.xml For more information see :ref:`topics-feed-exports` @@ -329,6 +350,7 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. + .. _faq-split-item: How to split an item into multiple items in an item pipeline? @@ -342,15 +364,15 @@ method for this purpose. For example:: from copy import deepcopy - from scrapy.item import BaseItem - + from itemadapter import is_item, ItemAdapter class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): for item in result: - if isinstance(item, (BaseItem, dict)): - for _ in range(item['multiply_by']): + if is_item(item): + adapter = ItemAdapter(item) + for _ in range(adapter['multiply_by']): yield deepcopy(item) Does Scrapy support IPv6 addresses? @@ -371,8 +393,33 @@ Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switch different reactor is possible by using the :setting:`TWISTED_REACTOR` setting. +.. _faq-stop-response-download: + +How can I cancel the download of a given response? +-------------------------------------------------- + +In some situations, it might be useful to stop the download of a certain response. +For instance, sometimes you can determine whether or not you need the full contents +of a response by inspecting its headers or the first bytes of its body. In that case, +you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received` +or :class:`~scrapy.signals.headers_received` signals and raising a +:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the +:ref:`topics-stop-response-download` topic for additional information and examples. + + +Running ``runspider`` I get ``error: No spider found in file: `` +-------------------------------------------------------------------------- + +This may happen if your Scrapy project has a spider module with a name that +conflicts with the name of one of the `Python standard library modules`_, such +as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed. +See :issue:`2680`. + + .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 +.. _Python standard library modules: https://docs.python.org/py-modindex.html +.. _Python package: https://pypi.org/ .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search +.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index a4343b7e0..433798aa8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -78,7 +78,6 @@ Basic concepts topics/settings topics/exceptions - :doc:`topics/commands` Learn about the command-line tool used to manage your Scrapy project. @@ -165,6 +164,8 @@ Solving specific problems topics/autothrottle topics/benchmarking topics/jobs + topics/coroutines + topics/asyncio :doc:`faq` Get answers to most frequently asked questions. @@ -205,6 +206,12 @@ Solving specific problems :doc:`topics/jobs` Learn how to pause and resume crawls for large spiders. +:doc:`topics/coroutines` + Use the :ref:`coroutine syntax `. + +:doc:`topics/asyncio` + Use :mod:`asyncio` and :mod:`asyncio`-powered libraries. + .. _extending-scrapy: Extending Scrapy @@ -220,6 +227,7 @@ Extending Scrapy topics/extensions topics/api topics/signals + topics/scheduler topics/exporters @@ -241,6 +249,9 @@ Extending Scrapy :doc:`topics/signals` See all available signals and how to work with them. +:doc:`topics/scheduler` + Understand the scheduler component. + :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 4739872fe..b8d3a16bc 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -4,12 +4,19 @@ Installation guide ================== +.. _faq-python-versions: + +Supported Python versions +========================= + +Scrapy requires Python 3.6+, either the CPython implementation (default) or +the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). + +.. _intro-install-scrapy: + Installing Scrapy ================= -Scrapy runs on Python 3.5 or above -under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). - If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows and macOS. @@ -23,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, +to avoid conflicting with your system packages. + Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, -to avoid conflicting with your system packages. - For more detailed and platform specifics instructions, as well as troubleshooting information, read on. @@ -63,10 +70,9 @@ In case of any trouble related to these dependencies, please refer to their respective installation instructions: * `lxml installation`_ -* `cryptography installation`_ +* :doc:`cryptography installation ` .. _lxml installation: https://lxml.de/installation.html -.. _cryptography installation: https://cryptography.io/en/latest/installation/ .. _intro-using-virtualenv: @@ -112,6 +118,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: conda install -c conda-forge scrapy +To install Scrapy on Windows using ``pip``: + +.. warning:: + This installation method requires “Microsoft Visual C++” for installing some + Scrapy dependencies, which demands significantly more disk space than Anaconda. + +#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. + +#. Run the Visual Studio Installer. + +#. Under the Workloads section, select **C++ build tools**. + +#. Check the installation details and make sure following packages are selected as optional components: + + * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) + + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) + +#. Install the Visual Studio Build Tools. + +Now, you should be able to :ref:`install Scrapy ` using ``pip``. .. _intro-install-ubuntu: @@ -207,8 +234,8 @@ For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. This means that these dependencies will be built during installation. -On macOS, you are likely to face an issue with building the Cryptography dependency. The -solution to this problem is described +On macOS, you are likely to face an issue with building the Cryptography +dependency. The solution to this problem is described `here `_, that is to ``brew install openssl`` and then export the flags that this command recommends (only needed when installing Scrapy). Installing on Linux has no special @@ -259,10 +286,10 @@ For details, see `Issue #2473 `_. .. _cryptography: https://cryptography.io/en/latest/ .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ .. _setuptools: https://pypi.python.org/pypi/setuptools -.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Scrapinghub: https://scrapinghub.com .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html +.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio +.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 01986b594..405bf845d 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -4,7 +4,7 @@ Scrapy at a glance ================== -Scrapy is an application framework for crawling web sites and extracting +Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting structured data which can be used for a wide range of useful applications, like data mining, information processing or historical archival. @@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination:: if next_page is not None: yield response.follow(next_page, self.parse) - Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider quotes_spider.py -o quotes.json + scrapy runspider quotes_spider.py -o quotes.jl +When this finishes you will have in the ``quotes.jl`` file a list of the +quotes in JSON Lines format, containing text and author, looking like this:: -When this finishes you will have in the ``quotes.json`` file a list of the -quotes in JSON format, containing text and author, looking like this (reformatted -here for better readability):: - - [{ - "author": "Jane Austen", - "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" - }, - { - "author": "Groucho Marx", - "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" - }, - { - "author": "Steve Martin", - "text": "\u201cA day without sunshine is like, you know, night.\u201d" - }, - ...] + {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} + {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} + {"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"} + ... What just happened? diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 1768badbb..ca5856881 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -25,16 +25,16 @@ Scrapy. If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource. If you're new to programming and want to start with Python, the following books -may be useful to you: +may be useful to you: * `Automate the Boring Stuff With Python`_ -* `How To Think Like a Computer Scientist`_ +* `How To Think Like a Computer Scientist`_ -* `Learn Python 3 The Hard Way`_ +* `Learn Python 3 The Hard Way`_ You can also take a look at `this list of Python resources for non-programmers`_, -as well as the `suggested resources in the learnpython-subreddit`_. +as well as the `suggested resources in the learnpython-subreddit`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers @@ -62,7 +62,7 @@ This will create a ``tutorial`` directory with the following contents:: __init__.py items.py # project items definition file - + middlewares.py # project middlewares file pipelines.py # project pipelines file @@ -78,7 +78,7 @@ Our first Spider Spiders are classes that you define and that Scrapy uses to scrape information from a website (or a group of websites). They must subclass -:class:`~scrapy.spiders.Spider` and define the initial requests to make, +:class:`~scrapy.Spider` and define the initial requests to make, optionally how to follow links in the pages, and how to parse the downloaded page content to extract data. @@ -101,32 +101,32 @@ This is the code for our first Spider. Save it in a file named def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) - self.log('Saved file %s' % filename) + self.log(f'Saved file {filename}') -As you can see, our Spider subclasses :class:`scrapy.Spider ` +As you can see, our Spider subclasses :class:`scrapy.Spider ` and defines some attributes and methods: -* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be +* :attr:`~scrapy.Spider.name`: identifies the Spider. It must be unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of +* :meth:`~scrapy.Spider.start_requests`: must return an iterable of Requests (you can return a list of requests or write a generator function) which the Spider will begin to crawl from. Subsequent requests will be generated successively from these initial requests. -* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle +* :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter is an instance of :class:`~scrapy.http.TextResponse` that holds the page content and has further helpful methods to handle it. - The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting + The :meth:`~scrapy.Spider.parse` method usually parses the response, extracting the scraped data as dicts and also finding new URLs to - follow and creating new requests (:class:`~scrapy.http.Request`) from them. + follow and creating new requests (:class:`~scrapy.Request`) from them. How to run our spider --------------------- @@ -162,7 +162,7 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects +Scrapy schedules the :class:`scrapy.Request ` objects returned by the ``start_requests`` method of the Spider. Upon receiving a response for each one, it instantiates :class:`~scrapy.http.Response` objects and calls the callback method associated with the request (in this case, the @@ -171,11 +171,11 @@ and calls the callback method associated with the request (in this case, the A shortcut to the start_requests method --------------------------------------- -Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute +Instead of implementing a :meth:`~scrapy.Spider.start_requests` method +that generates :class:`scrapy.Request ` objects from URLs, +you can just define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests +of :meth:`~scrapy.Spider.start_requests` to create the initial requests for your spider:: import scrapy @@ -190,13 +190,13 @@ for your spider:: def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) -The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each +The :meth:`~scrapy.Spider.parse` method will be called to handle each of the requests for those URLs, even though we haven't explicitly told Scrapy -to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's +to do so. This happens because :meth:`~scrapy.Spider.parse` is Scrapy's default callback method, which is called for requests without an explicitly assigned callback. @@ -248,7 +248,7 @@ object: The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of -:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements +:class:`~scrapy.Selector` objects that wrap around XML/HTML elements and allow you to run further queries to fine-grain the selection or extract the data. @@ -277,9 +277,19 @@ As an alternative, you could've written: >>> response.css('title::text')[0].get() 'Quotes to Scrape' -However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList` -instance avoids an ``IndexError`` and returns ``None`` when it doesn't -find any element matching the selection. +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +raise an :exc:`IndexError` exception if there are no results:: + + >>> response.css('noelement')[0].get() + Traceback (most recent call last): + ... + IndexError: list index out of range + +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +if there are no results:: + +>>> response.css("noelement").get() There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail @@ -287,8 +297,8 @@ to be scraped, you can at least get **some** data. Besides the :meth:`~scrapy.selector.SelectorList.getall` and :meth:`~scrapy.selector.SelectorList.get` methods, you can also use -the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular -expressions`_: +the :meth:`~scrapy.selector.SelectorList.re` method to extract using +:doc:`regular expressions `: >>> response.css('title::text').re(r'Quotes.*') ['Quotes to Scrape'] @@ -305,7 +315,6 @@ with a selector (see :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for visually selected elements, which works in many browsers. -.. _regular expressions: https://docs.python.org/3/library/re.html .. _Selector Gadget: https://selectorgadget.com/ @@ -406,8 +415,6 @@ to get all of them: from sys import version_info -.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output") - Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary: @@ -465,16 +472,15 @@ Storing the scraped data The simplest way to store the scraped data is by using :ref:`Feed exports `, with the following command:: - scrapy crawl quotes -o quotes.json + scrapy crawl quotes -O quotes.json -That will generate an ``quotes.json`` file containing all scraped items, +That will generate a ``quotes.json`` file containing all scraped items, serialized in `JSON`_. -For historic reasons, Scrapy appends to a given file instead of overwriting -its contents. If you run this command twice without removing the file -before the second time, you'll end up with a broken JSON file. - -You can also use other formats, like `JSON Lines`_:: +The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead +to append new content to any existing file. However, appending to a JSON file +makes the file contents invalid JSON. When appending to a file, consider +using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jl @@ -674,7 +680,7 @@ the pagination links with the ``parse`` callback as we saw before. Here we're passing callbacks to :meth:`response.follow_all ` as positional arguments to make the code shorter; it also works for -:class:`~scrapy.http.Request`. +:class:`~scrapy.Request`. The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. @@ -705,7 +711,7 @@ Using spider arguments You can provide command line arguments to your spiders by using the ``-a`` option when running them:: - scrapy crawl quotes -o quotes-humor.json -a tag=humor + scrapy crawl quotes -O quotes-humor.json -a tag=humor These arguments are passed to the Spider's ``__init__`` method and become spider attributes by default. diff --git a/docs/news.rst b/docs/news.rst index c1daedaf2..2afe318f6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,10 +1,1556 @@ +.. note:: + .. versionchanged:: VERSION + + The Twisted reactor is now installed when + :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a + :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, + :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now + honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions + they are silently ignored when set there and you need to set these settings + in some other way. + + +.. note:: + .. versionchanged:: VERSION + + Previously this setting had no effect in a spider + :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but + if you :ref:`run several spiders in one process `, + they must not have different values for this setting, because they will use + a single reactor instance. + + .. _news: Release notes ============= -.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0, - planned for Q4 2019 or Q1 2020, will support **Python 3 only**. +.. _release-2.5.1: + +Scrapy 2.5.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + +.. _release-2.5.0: + +Scrapy 2.5.0 (2021-04-06) +------------------------- + +Highlights: + +- Official Python 3.9 support + +- Experimental :ref:`HTTP/2 support ` + +- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function + to retry requests from spider callbacks + +- New :class:`~scrapy.signals.headers_received` signal that allows stopping + downloads early + +- New :class:`Response.protocol ` attribute + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and + had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`. + (:issue:`4901`) + +- Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment + variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`) + + +Deprecations +~~~~~~~~~~~~ + +- The :mod:`scrapy.utils.py36` module is now deprecated in favor of + :mod:`scrapy.utils.asyncgen`. (:issue:`4900`) + + +New features +~~~~~~~~~~~~ + +- Experimental :ref:`HTTP/2 support ` through a new download handler + that can be assigned to the ``https`` protocol in the + :setting:`DOWNLOAD_HANDLERS` setting. + (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) + +- The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request` + function may be used from spider callbacks or middlewares to handle the + retrying of a request beyond the scenarios that + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports. + (:issue:`3590`, :issue:`3685`, :issue:`4902`) + +- The new :class:`~scrapy.signals.headers_received` signal gives early access + to response headers and allows :ref:`stopping downloads + `. + (:issue:`1772`, :issue:`4897`) + +- The new :attr:`Response.protocol ` + attribute gives access to the string that identifies the protocol used to + download a response. (:issue:`4878`) + +- :ref:`Stats ` now include the following entries that indicate + the number of successes and failures in storing + :ref:`feeds `:: + + feedexport/success_count/ + feedexport/failed_count/ + + Where ```` is the feed storage backend class name, such as + :class:`~scrapy.extensions.feedexport.FileFeedStorage` or + :class:`~scrapy.extensions.feedexport.FTPFeedStorage`. + + (:issue:`3947`, :issue:`4850`) + +- The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider + middleware now logs ignored URLs with ``INFO`` :ref:`logging level + ` instead of ``DEBUG``, and it now includes the following entry + into :ref:`stats ` to keep track of the number of ignored + URLs:: + + urllength/request_ignored_count + + (:issue:`5036`) + +- The + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + downloader middleware now logs the number of decompressed responses and the + total count of resulting bytes:: + + httpcompression/response_bytes + httpcompression/response_count + + (:issue:`4797`, :issue:`4799`) + + +Bug fixes +~~~~~~~~~ + +- Fixed installation on PyPy installing PyDispatcher in addition to + PyPyDispatcher, which could prevent Scrapy from working depending on which + package got imported. (:issue:`4710`, :issue:`4814`) + +- When inspecting a callback to check if it is a generator that also returns + a value, an exception is no longer raised if the callback has a docstring + with lower indentation than the following code. + (:issue:`4477`, :issue:`4935`) + +- The `Content-Length `_ + header is no longer omitted from responses when using the default, HTTP/1.1 + download handler (see :setting:`DOWNLOAD_HANDLERS`). + (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`) + +- Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False`` + now has the same effect as not setting it at all, instead of having the + same effect as setting it to ``True``. + (:issue:`3851`, :issue:`4694`) + + +Documentation +~~~~~~~~~~~~~ + +- Added instructions to :ref:`install Scrapy in Windows using pip + `. + (:issue:`4715`, :issue:`4736`) + +- Logging documentation now includes :ref:`additional ways to filter logs + `. + (:issue:`4216`, :issue:`4257`, :issue:`4965`) + +- Covered how to deal with long lists of allowed domains in the :ref:`FAQ + `. (:issue:`2263`, :issue:`3667`) + +- Covered scrapy-bench_ in :ref:`benchmarking`. + (:issue:`4996`, :issue:`5016`) + +- Clarified that one :ref:`extension ` instance is created + per crawler. + (:issue:`5014`) + +- Fixed some errors in examples. + (:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`, + :issue:`5008`) + +- Fixed some external links, typos, and so on. + (:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`, + :issue:`5063`) + +- The :ref:`list of Request.meta keys ` is now sorted + alphabetically. + (:issue:`5061`, :issue:`5065`) + +- Updated references to Scrapinghub, which is now called Zyte. + (:issue:`4973`, :issue:`5072`) + +- Added a mention to contributors in the README. (:issue:`4956`) + +- Reduced the top margin of lists. (:issue:`4974`) + + +Quality Assurance +~~~~~~~~~~~~~~~~~ + +- Made Python 3.9 support official (:issue:`4757`, :issue:`4759`) + +- Extended typing hints (:issue:`4895`) + +- Fixed deprecated uses of the Twisted API. + (:issue:`4940`, :issue:`4950`, :issue:`5073`) + +- Made our tests run with the new pip resolver. + (:issue:`4710`, :issue:`4814`) + +- Added tests to ensure that :ref:`coroutine support ` + is tested. (:issue:`4987`) + +- Migrated from Travis CI to GitHub Actions. (:issue:`4924`) + +- Fixed CI issues. + (:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`, + :issue:`5053`) + +- Implemented code refactorings, style fixes and cleanups. + (:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`) + + +.. _release-2.4.1: + +Scrapy 2.4.1 (2020-11-17) +------------------------- + +- Fixed :ref:`feed exports ` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`) + +- Fixed the AsyncIO event loop handling, which could make code hang + (:issue:`4855`, :issue:`4872`) + +- Fixed the IPv6-capable DNS resolver + :class:`~scrapy.resolver.CachingHostnameResolver` for download handlers + that call + :meth:`reactor.resolve ` + (:issue:`4802`, :issue:`4803`) + +- Fixed the output of the :command:`genspider` command showing placeholders + instead of the import path of the generated spider module (:issue:`4874`) + +- Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`, + :issue:`4876`) + + +.. _release-2.4.0: + +Scrapy 2.4.0 (2020-10-11) +------------------------- + +Highlights: + +* Python 3.5 support has been dropped. + +* The ``file_path`` method of :ref:`media pipelines ` + can now access the source :ref:`item `. + + This allows you to set a download file path based on item data. + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` + +* You can now choose whether :ref:`feed exports ` + overwrite or append to the output file. + + For example, when using the :command:`crawl` or :command:`runspider` + commands, you can use the ``-O`` option instead of ``-o`` to overwrite the + output file. + +* Zstd-compressed responses are now supported if zstandard_ is installed. + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +* Python 3.6 or greater is now required; support for Python 3.5 has been + dropped + + As a result: + + - When using PyPy, PyPy 7.2.0 or greater :ref:`is now required + ` + + - For Amazon S3 storage support in :ref:`feed exports + ` or :ref:`media pipelines + `, botocore_ 1.4.87 or greater is now required + + - To use the :ref:`images pipeline `, Pillow_ 4.0.0 or + greater is now required + + (:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`, + :issue:`4764`) + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again + discards cookies defined in :attr:`Request.headers + `. + + We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it + was reported that the current implementation could break existing code. + + If you need to set cookies for a request, use the :class:`Request.cookies + ` parameter. + + A future version of Scrapy will include a new, better implementation of the + reverted bug fix. + + (:issue:`4717`, :issue:`4823`) + + +.. _2.4-deprecation-removals: + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the + values of ``access_key`` and ``secret_key`` from the running project + settings when they are not passed to its ``__init__`` method; you must + either pass those parameters to its ``__init__`` method or use + :class:`S3FeedStorage.from_crawler + ` + (:issue:`4356`, :issue:`4411`, :issue:`4688`) + +* :attr:`Rule.process_request ` + no longer admits callables which expect a single ``request`` parameter, + rather than both ``request`` and ``response`` (:issue:`4818`) + + +Deprecations +~~~~~~~~~~~~ + +* In custom :ref:`media pipelines `, signatures that + do not accept a keyword-only ``item`` parameter in any of the methods that + :ref:`now support this parameter ` are now + deprecated (:issue:`4628`, :issue:`4686`) + +* In custom :ref:`feed storage backend classes `, + ``__init__`` method signatures that do not accept a keyword-only + ``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`, + :issue:`4512`) + +* The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated + (:issue:`4684`, :issue:`4701`) + +* The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use + :func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`, + :issue:`4776`) + + +New features +~~~~~~~~~~~~ + +.. _media-pipeline-item-parameter: + +* The following methods of :ref:`media pipelines ` now + accept an ``item`` keyword-only parameter containing the source + :ref:`item `: + + - In :class:`scrapy.pipelines.files.FilesPipeline`: + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_path` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download` + + - In :class:`scrapy.pipelines.images.ImagesPipeline`: + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download` + + (:issue:`4628`, :issue:`4686`) + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` (:issue:`4606`, :issue:`4768`) + +* :ref:`Feed exports ` gained overwrite support: + + * When using the :command:`crawl` or :command:`runspider` commands, you + can use the ``-O`` option instead of ``-o`` to overwrite the output + file + + * You can use the ``overwrite`` key in the :setting:`FEEDS` setting to + configure whether to overwrite the output file (``True``) or append to + its content (``False``) + + * The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage + backend classes ` now receive a new keyword-only + parameter, ``feed_options``, which is a dictionary of :ref:`feed + options ` + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* Zstd-compressed responses are now supported if zstandard_ is installed + (:issue:`4831`) + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead (:issue:`3870`, :issue:`3873`). + + This includes also settings where only part of its value is made of an + import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or + :setting:`DOWNLOAD_HANDLERS`. + +* :ref:`Downloader middlewares ` can now + override :class:`response.request `. + + If a :ref:`downloader middleware ` returns + a :class:`~scrapy.http.Response` object from + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` + or + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` + with a custom :class:`~scrapy.http.Request` object assigned to + :class:`response.request `: + + - The response is handled by the callback of that custom + :class:`~scrapy.http.Request` object, instead of being handled by the + callback of the original :class:`~scrapy.http.Request` object + + - That custom :class:`~scrapy.http.Request` object is now sent as the + ``request`` argument to the :signal:`response_received` signal, instead + of the original :class:`~scrapy.http.Request` object + + (:issue:`4529`, :issue:`4632`) + +* When using the :ref:`FTP feed storage backend `: + + - It is now possible to set the new ``overwrite`` :ref:`feed option + ` to ``False`` to append to an existing file instead of + overwriting it + + - The FTP password can now be omitted if it is not necessary + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now + supports an ``errors`` parameter to indicate how to handle encoding errors + (:issue:`4755`) + +* When :ref:`using asyncio `, it is now possible to + :ref:`set a custom asyncio loop ` (:issue:`4306`, + :issue:`4414`) + +* Serialized requests (see :ref:`topics-jobs`) now support callbacks that are + spider methods that delegate on other callable (:issue:`4756`) + +* When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged + message is now a warning, instead of an error (:issue:`3874`, + :issue:`3886`, :issue:`4752`) + + +Bug fixes +~~~~~~~~~ + +* The :command:`genspider` command no longer overwrites existing files + unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`, + :issue:`4623`) + +* Cookies with an empty value are no longer considered invalid cookies + (:issue:`4772`) + +* The :command:`runspider` command now supports files with the ``.pyw`` file + extension (:issue:`4643`, :issue:`4646`) + +* The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + middleware now simply ignores unsupported proxy values (:issue:`3331`, + :issue:`4778`) + +* Checks for generator callbacks with a ``return`` statement no longer warn + about ``return`` statements in nested functions (:issue:`4720`, + :issue:`4721`) + +* The system file mode creation mask no longer affects the permissions of + files generated using the :command:`startproject` command (:issue:`4722`) + +* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names + (:issue:`861`, :issue:`4746`) + +* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can + work when using a headless browser (:issue:`4835`) + + +Documentation +~~~~~~~~~~~~~ + +* The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`, + :issue:`4724`) + +* Improved the documentation of + :ref:`link extractors ` with an usage example from + a spider callback and reference documentation for the + :class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`) + +* Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the + :class:`~scrapy.extensions.closespider.CloseSpider` extension + (:issue:`4836`) + +* Removed references to Python 2’s ``unicode`` type (:issue:`4547`, + :issue:`4703`) + +* We now have an :ref:`official deprecation policy ` + (:issue:`4705`) + +* Our :ref:`documentation policies ` now cover usage + of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged` + directives, and we have removed usages referencing Scrapy 1.4.0 and earlier + versions (:issue:`3971`, :issue:`4310`) + +* Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`, + :issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Extended typing hints (:issue:`4243`, :issue:`4691`) + +* Added tests for the :command:`check` command (:issue:`4663`) + +* Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`) + +* Improved Windows test coverage (:issue:`4723`) + +* Switched to :ref:`formatted string literals ` where possible + (:issue:`4307`, :issue:`4324`, :issue:`4672`) + +* Modernized :func:`super` usage (:issue:`4707`) + +* Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`, + :issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`, + :issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`, + :issue:`4820`, :issue:`4822`, :issue:`4839`) + + +.. _release-2.3.0: + +Scrapy 2.3.0 (2020-08-04) +------------------------- + +Highlights: + +* :ref:`Feed exports ` now support :ref:`Google Cloud + Storage ` as a storage backend + +* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver + output items in batches of up to the specified number of items. + + It also serves as a workaround for :ref:`delayed file delivery + `, which causes Scrapy to only start item delivery + after the crawl has finished when using certain storage backends + (:ref:`S3 `, :ref:`FTP `, + and now :ref:`GCS `). + +* The base implementation of :ref:`item loaders ` has been + moved into a separate library, :doc:`itemloaders `, + allowing usage from outside Scrapy and a separate release schedule + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* Removed the following classes and their parent modules from + ``scrapy.linkextractors``: + + * ``htmlparser.HtmlParserLinkExtractor`` + * ``regex.RegexLinkExtractor`` + * ``sgml.BaseSgmlLinkExtractor`` + * ``sgml.SgmlLinkExtractor`` + + Use + :class:`LinkExtractor ` + instead (:issue:`4356`, :issue:`4679`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated + (:issue:`4683`) + + +New features +~~~~~~~~~~~~ + +* :ref:`Feed exports ` support :ref:`Google Cloud + Storage ` (:issue:`685`, :issue:`3608`) + +* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries + (:issue:`4250`, :issue:`4434`) + +* The :command:`parse` command now allows specifying an output file + (:issue:`4317`, :issue:`4377`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support + ``--data-raw`` (:issue:`4612`) + +* A ``parse`` callback may now be used in built-in spider subclasses, such + as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`, + :issue:`781`, :issue:`4254` ) + + +Bug fixes +~~~~~~~~~ + +* Fixed the :ref:`CSV exporting ` of + :ref:`dataclass items ` and :ref:`attr.s items + ` (:issue:`4667`, :issue:`4668`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request + method to ``POST`` when a request body is specified and no request method + is specified (:issue:`4612`) + +* The processing of ANSI escape sequences in enabled in Windows 10.0.14393 + and later, where it is required for colored output (:issue:`4393`, + :issue:`4403`) + + +Documentation +~~~~~~~~~~~~~ + +* Updated the `OpenSSL cipher list format`_ link in the documentation about + the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`) + +* Simplified the code example in :ref:`topics-loaders-dataclass` + (:issue:`4652`) + +.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* The base implementation of :ref:`item loaders ` has been + moved into :doc:`itemloaders ` (:issue:`4005`, + :issue:`4516`) + +* Fixed a silenced error in some scheduler tests (:issue:`4644`, + :issue:`4645`) + +* Renewed the localhost certificate used for SSL tests (:issue:`4650`) + +* Removed cookie-handling code specific to Python 2 (:issue:`4682`) + +* Stopped using Python 2 unicode literal syntax (:issue:`4704`) + +* Stopped using a backlash for line continuation (:issue:`4673`) + +* Removed unneeded entries from the MyPy exception list (:issue:`4690`) + +* Automated tests now pass on Windows as part of our continuous integration + system (:issue:`4458`) + +* Automated tests now pass on the latest PyPy version for supported Python + versions in our continuous integration system (:issue:`4504`) + + +.. _release-2.2.1: + +Scrapy 2.2.1 (2020-07-17) +------------------------- + +* The :command:`startproject` command no longer makes unintended changes to + the permissions of files in the destination folder, such as removing + execution permissions (:issue:`4662`, :issue:`4666`) + + +.. _release-2.2.0: + +Scrapy 2.2.0 (2020-06-24) +------------------------- + +Highlights: + +* Python 3.5.2+ is required now +* :ref:`dataclass objects ` and + :ref:`attrs objects ` are now valid :ref:`item types + ` +* New :meth:`TextResponse.json ` method +* New :signal:`bytes_received` signal that allows canceling response download +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` fixes + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Support for Python 3.5.0 and 3.5.1 has been dropped; Scrapy now refuses to + run with a Python version lower than 3.5.2, which introduced + :class:`typing.Type` (:issue:`4615`) + + +Deprecations +~~~~~~~~~~~~ + +* :meth:`TextResponse.body_as_unicode + ` is now deprecated, use + :attr:`TextResponse.text ` instead + (:issue:`4546`, :issue:`4555`, :issue:`4579`) + +* :class:`scrapy.item.BaseItem` is now deprecated, use + :class:`scrapy.item.Item` instead (:issue:`4534`) + + +New features +~~~~~~~~~~~~ + +* :ref:`dataclass objects ` and + :ref:`attrs objects ` are now valid :ref:`item types + `, and a new itemadapter_ library makes it easy to + write code that :ref:`supports any item type ` + (:issue:`2749`, :issue:`2807`, :issue:`3761`, :issue:`3881`, :issue:`4642`) + +* A new :meth:`TextResponse.json ` method + allows to deserialize JSON responses (:issue:`2444`, :issue:`4460`, + :issue:`4574`) + +* A new :signal:`bytes_received` signal allows monitoring response download + progress and :ref:`stopping downloads ` + (:issue:`4205`, :issue:`4559`) + +* The dictionaries in the result list of a :ref:`media pipeline + ` now include a new key, ``status``, which indicates + if the file was downloaded or, if the file was not downloaded, why it was + not downloaded; see :meth:`FilesPipeline.get_media_requests + ` for more + information (:issue:`2893`, :issue:`4486`) + +* When using :ref:`Google Cloud Storage ` for + a :ref:`media pipeline `, a warning is now logged if + the configured credentials do not grant the required permissions + (:issue:`4346`, :issue:`4508`) + +* :ref:`Link extractors ` are now serializable, + as long as you do not use :ref:`lambdas ` for parameters; for + example, you can now pass link extractors in :attr:`Request.cb_kwargs + ` or + :attr:`Request.meta ` when :ref:`persisting + scheduled requests ` (:issue:`4554`) + +* Upgraded the :ref:`pickle protocol ` that Scrapy uses + from protocol 2 to protocol 4, improving serialization capabilities and + performance (:issue:`4135`, :issue:`4541`) + +* :func:`scrapy.utils.misc.create_instance` now raises a :exc:`TypeError` + exception if the resulting instance is ``None`` (:issue:`4528`, + :issue:`4532`) + +.. _itemadapter: https://github.com/scrapy/itemadapter + + +Bug fixes +~~~~~~~~~ + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer + discards cookies defined in :attr:`Request.headers + ` (:issue:`1992`, :issue:`2400`) + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer + re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter + of the ``__init__`` method of :class:`~scrapy.http.Request` + (:issue:`2400`, :issue:`3575`) + +* When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is + ``False`` and the crawl yields no items, Scrapy no longer stops feed + exports after the first URI (:issue:`4621`, :issue:`4626`) + +* :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine + syntax ` no longer need to return an iterable, and may + instead return a :class:`~scrapy.http.Request` object, an + :ref:`item `, or ``None`` (:issue:`4609`) + +* The :command:`startproject` command now ensures that the generated project + folders and files have the right permissions (:issue:`4604`) + +* Fix a :exc:`KeyError` exception being sometimes raised from + :class:`scrapy.utils.datatypes.LocalWeakReferencedCache` (:issue:`4597`, + :issue:`4599`) + +* When :setting:`FEEDS` defines multiple URIs, log messages about items being + stored now contain information from the corresponding feed, instead of + always containing information about only one of the feeds (:issue:`4619`, + :issue:`4629`) + + +Documentation +~~~~~~~~~~~~~ + +* Added a new section about :ref:`accessing cb_kwargs from errbacks + ` (:issue:`4598`, :issue:`4634`) + +* Covered chompjs_ in :ref:`topics-parsing-javascript` (:issue:`4556`, + :issue:`4562`) + +* Removed from :doc:`topics/coroutines` the warning about the API being + experimental (:issue:`4511`, :issue:`4513`) + +* Removed references to unsupported versions of :doc:`Twisted + ` (:issue:`4533`) + +* Updated the description of the :ref:`screenshot pipeline example + `, which now uses :doc:`coroutine syntax + ` instead of returning a + :class:`~twisted.internet.defer.Deferred` (:issue:`4514`, :issue:`4593`) + +* Removed a misleading import line from the + :func:`scrapy.utils.log.configure_logging` code example (:issue:`4510`, + :issue:`4587`) + +* The display-on-hover behavior of internal documentation references now also + covers links to :ref:`commands `, :attr:`Request.meta + ` keys, :ref:`settings ` and + :ref:`signals ` (:issue:`4495`, :issue:`4563`) + +* It is again possible to download the documentation for offline reading + (:issue:`4578`, :issue:`4585`) + +* Removed backslashes preceding ``*args`` and ``**kwargs`` in some function + and method signatures (:issue:`4592`, :issue:`4596`) + +.. _chompjs: https://github.com/Nykakin/chompjs + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Adjusted the code base further to our :ref:`style guidelines + ` (:issue:`4237`, :issue:`4525`, :issue:`4538`, + :issue:`4539`, :issue:`4540`, :issue:`4542`, :issue:`4543`, :issue:`4544`, + :issue:`4545`, :issue:`4557`, :issue:`4558`, :issue:`4566`, :issue:`4568`, + :issue:`4572`) + +* Removed remnants of Python 2 support (:issue:`4550`, :issue:`4553`, + :issue:`4568`) + +* Improved code sharing between the :command:`crawl` and :command:`runspider` + commands (:issue:`4548`, :issue:`4552`) + +* Replaced ``chain(*iterable)`` with ``chain.from_iterable(iterable)`` + (:issue:`4635`) + +* You may now run the :mod:`asyncio` tests with Tox on any Python version + (:issue:`4521`) + +* Updated test requirements to reflect an incompatibility with pytest 5.4 and + 5.4.1 (:issue:`4588`) + +* Improved :class:`~scrapy.spiderloader.SpiderLoader` test coverage for + scenarios involving duplicate spider names (:issue:`4549`, :issue:`4560`) + +* Configured Travis CI to also run the tests with Python 3.5.2 + (:issue:`4518`, :issue:`4615`) + +* Added a `Pylint `_ job to Travis CI + (:issue:`3727`) + +* Added a `Mypy `_ job to Travis CI (:issue:`4637`) + +* Made use of set literals in tests (:issue:`4573`) + +* Cleaned up the Travis CI configuration (:issue:`4517`, :issue:`4519`, + :issue:`4522`, :issue:`4537`) + + +.. _release-2.1.0: + +Scrapy 2.1.0 (2020-04-24) +------------------------- + +Highlights: + +* New :setting:`FEEDS` setting to export to multiple feeds +* New :attr:`Response.ip_address ` attribute + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :exc:`AssertionError` exceptions triggered by :ref:`assert ` + statements have been replaced by new exception types, to support running + Python in optimized mode (see :option:`-O`) without changing Scrapy’s + behavior in any unexpected ways. + + If you catch an :exc:`AssertionError` exception from Scrapy, update your + code to catch the corresponding new exception. + + (:issue:`4440`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* The ``LOG_UNSERIALIZABLE_REQUESTS`` setting is no longer supported, use + :setting:`SCHEDULER_DEBUG` instead (:issue:`4385`) + +* The ``REDIRECT_MAX_METAREFRESH_DELAY`` setting is no longer supported, use + :setting:`METAREFRESH_MAXDELAY` instead (:issue:`4385`) + +* The :class:`~scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` + middleware has been removed, including the entire + :class:`scrapy.downloadermiddlewares.chunked` module; chunked transfers + work out of the box (:issue:`4431`) + +* The ``spiders`` property has been removed from + :class:`~scrapy.crawler.Crawler`, use :class:`CrawlerRunner.spider_loader + ` or instantiate + :setting:`SPIDER_LOADER_CLASS` with your settings instead (:issue:`4398`) + +* The ``MultiValueDict``, ``MultiValueDictKeyError``, and ``SiteNode`` + classes have been removed from :mod:`scrapy.utils.datatypes` + (:issue:`4400`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``FEED_FORMAT`` and ``FEED_URI`` settings have been deprecated in + favor of the new :setting:`FEEDS` setting (:issue:`1336`, :issue:`3858`, + :issue:`4507`) + + +New features +~~~~~~~~~~~~ + +* A new setting, :setting:`FEEDS`, allows configuring multiple output feeds + with different settings each (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The :command:`crawl` and :command:`runspider` commands now support multiple + ``-o`` parameters (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The :command:`crawl` and :command:`runspider` commands now support + specifying an output format by appending ``:`` to the output file + (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The new :attr:`Response.ip_address ` + attribute gives access to the IP address that originated a response + (:issue:`3903`, :issue:`3940`) + +* A warning is now issued when a value in + :attr:`~scrapy.spiders.Spider.allowed_domains` includes a port + (:issue:`50`, :issue:`3198`, :issue:`4413`) + +* Zsh completion now excludes used option aliases from the completion list + (:issue:`4438`) + + +Bug fixes +~~~~~~~~~ + +* :ref:`Request serialization ` no longer breaks for + callbacks that are spider attributes which are assigned a function with a + different name (:issue:`4500`) + +* ``None`` values in :attr:`~scrapy.spiders.Spider.allowed_domains` no longer + cause a :exc:`TypeError` exception (:issue:`4410`) + +* Zsh completion no longer allows options after arguments (:issue:`4438`) + +* zope.interface 5.0.0 and later versions are now supported + (:issue:`4447`, :issue:`4448`) + +* :meth:`Spider.make_requests_from_url + `, deprecated in Scrapy + 1.4.0, now issues a warning when used (:issue:`4412`) + + +Documentation +~~~~~~~~~~~~~ + +* Improved the documentation about signals that allow their handlers to + return a :class:`~twisted.internet.defer.Deferred` (:issue:`4295`, + :issue:`4390`) + +* Our PyPI entry now includes links for our documentation, our source code + repository and our issue tracker (:issue:`4456`) + +* Covered the `curl2scrapy `_ + service in the documentation (:issue:`4206`, :issue:`4455`) + +* Removed references to the Guppy library, which only works in Python 2 + (:issue:`4285`, :issue:`4343`) + +* Extended use of InterSphinx to link to Python 3 documentation + (:issue:`4444`, :issue:`4445`) + +* Added support for Sphinx 3.0 and later (:issue:`4475`, :issue:`4480`, + :issue:`4496`, :issue:`4503`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Removed warnings about using old, removed settings (:issue:`4404`) + +* Removed a warning about importing + :class:`~twisted.internet.testing.StringTransport` from + ``twisted.test.proto_helpers`` in Twisted 19.7.0 or newer (:issue:`4409`) + +* Removed outdated Debian package build files (:issue:`4384`) + +* Removed :class:`object` usage as a base class (:issue:`4430`) + +* Removed code that added support for old versions of Twisted that we no + longer support (:issue:`4472`) + +* Fixed code style issues (:issue:`4468`, :issue:`4469`, :issue:`4471`, + :issue:`4481`) + +* Removed :func:`twisted.internet.defer.returnValue` calls (:issue:`4443`, + :issue:`4446`, :issue:`4489`) + + +.. _release-2.0.1: + +Scrapy 2.0.1 (2020-03-18) +------------------------- + +* :meth:`Response.follow_all ` now supports + an empty URL iterable as input (:issue:`4408`, :issue:`4420`) + +* Removed top-level :mod:`~twisted.internet.reactor` imports to prevent + errors about the wrong Twisted reactor being installed when setting a + different Twisted reactor using :setting:`TWISTED_REACTOR` (:issue:`4401`, + :issue:`4406`) + +* Fixed tests (:issue:`4422`) + + +.. _release-2.0.0: + +Scrapy 2.0.0 (2020-03-03) +------------------------- + +Highlights: + +* Python 2 support has been removed +* :doc:`Partial ` :ref:`coroutine syntax ` support + and :doc:`experimental ` :mod:`asyncio` support +* New :meth:`Response.follow_all ` method +* :ref:`FTP support ` for media pipelines +* New :attr:`Response.certificate ` + attribute +* IPv6 support through :setting:`DNS_RESOLVER` + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Python 2 support has been removed, following `Python 2 end-of-life on + January 1, 2020`_ (:issue:`4091`, :issue:`4114`, :issue:`4115`, + :issue:`4121`, :issue:`4138`, :issue:`4231`, :issue:`4242`, :issue:`4304`, + :issue:`4309`, :issue:`4373`) + +* Retry gaveups (see :setting:`RETRY_TIMES`) are now logged as errors instead + of as debug information (:issue:`3171`, :issue:`3566`) + +* File extensions that + :class:`LinkExtractor ` + ignores by default now also include ``7z``, ``7zip``, ``apk``, ``bz2``, + ``cdr``, ``dmg``, ``ico``, ``iso``, ``tar``, ``tar.gz``, ``webm``, and + ``xz`` (:issue:`1837`, :issue:`2067`, :issue:`4066`) + +* The :setting:`METAREFRESH_IGNORE_TAGS` setting is now an empty list by + default, following web browser behavior (:issue:`3844`, :issue:`4311`) + +* The + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + now includes spaces after commas in the value of the ``Accept-Encoding`` + header that it sets, following web browser behavior (:issue:`4293`) + +* The ``__init__`` method of custom download handlers (see + :setting:`DOWNLOAD_HANDLERS`) or subclasses of the following downloader + handlers no longer receives a ``settings`` parameter: + + * :class:`scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler` + + * :class:`scrapy.core.downloader.handlers.file.FileDownloadHandler` + + Use the ``from_settings`` or ``from_crawler`` class methods to expose such + a parameter to your custom download handlers. + + (:issue:`4126`) + +* We have refactored the :class:`scrapy.core.scheduler.Scheduler` class and + related queue classes (see :setting:`SCHEDULER_PRIORITY_QUEUE`, + :setting:`SCHEDULER_DISK_QUEUE` and :setting:`SCHEDULER_MEMORY_QUEUE`) to + make it easier to implement custom scheduler queue classes. See + :ref:`2-0-0-scheduler-queue-changes` below for details. + +* Overridden settings are now logged in a different format. This is more in + line with similar information logged at startup (:issue:`4199`) + +.. _Python 2 end-of-life on January 1, 2020: https://www.python.org/doc/sunset-python-2/ + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* The :ref:`Scrapy shell ` no longer provides a `sel` proxy + object, use :meth:`response.selector ` + instead (:issue:`4347`) + +* LevelDB support has been removed (:issue:`4112`) + +* The following functions have been removed from :mod:`scrapy.utils.python`: + ``isbinarytext``, ``is_writable``, ``setattr_default``, ``stringify_dict`` + (:issue:`4362`) + + +Deprecations +~~~~~~~~~~~~ + +* Using environment variables prefixed with ``SCRAPY_`` to override settings + is deprecated (:issue:`4300`, :issue:`4374`, :issue:`4375`) + +* :class:`scrapy.linkextractors.FilteringLinkExtractor` is deprecated, use + :class:`scrapy.linkextractors.LinkExtractor + ` instead (:issue:`4045`) + +* The ``noconnect`` query string argument of proxy URLs is deprecated and + should be removed from proxy URLs (:issue:`4198`) + +* The :meth:`next ` method of + :class:`scrapy.utils.python.MutableChain` is deprecated, use the global + :func:`next` function or :meth:`MutableChain.__next__ + ` instead (:issue:`4153`) + + +New features +~~~~~~~~~~~~ + +* Added :doc:`partial support ` for Python’s + :ref:`coroutine syntax ` and :doc:`experimental support + ` for :mod:`asyncio` and :mod:`asyncio`-powered libraries + (:issue:`4010`, :issue:`4259`, :issue:`4269`, :issue:`4270`, :issue:`4271`, + :issue:`4316`, :issue:`4318`) + +* The new :meth:`Response.follow_all ` + method offers the same functionality as + :meth:`Response.follow ` but supports an + iterable of URLs as input and returns an iterable of requests + (:issue:`2582`, :issue:`4057`, :issue:`4286`) + +* :ref:`Media pipelines ` now support :ref:`FTP + storage ` (:issue:`3928`, :issue:`3961`) + +* The new :attr:`Response.certificate ` + attribute exposes the SSL certificate of the server as a + :class:`twisted.internet.ssl.Certificate` object for HTTPS responses + (:issue:`2726`, :issue:`4054`) + +* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support + (:issue:`1031`, :issue:`4227`) + +* A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring + the existing soft limit that pauses request downloads when the total + response data being processed is too high (:issue:`1410`, :issue:`3551`) + +* A new :setting:`TWISTED_REACTOR` setting allows customizing the + :mod:`~twisted.internet.reactor` that Scrapy uses, allowing to + :doc:`enable asyncio support ` or deal with a + :ref:`common macOS issue ` (:issue:`2905`, + :issue:`4294`) + +* Scheduler disk and memory queues may now use the class methods + ``from_crawler`` or ``from_settings`` (:issue:`3884`) + +* The new :attr:`Response.cb_kwargs ` + attribute serves as a shortcut for :attr:`Response.request.cb_kwargs + ` (:issue:`4331`) + +* :meth:`Response.follow ` now supports a + ``flags`` parameter, for consistency with :class:`~scrapy.http.Request` + (:issue:`4277`, :issue:`4279`) + +* :ref:`Item loader processors ` can now be + regular functions, they no longer need to be methods (:issue:`3899`) + +* :class:`~scrapy.spiders.Rule` now accepts an ``errback`` parameter + (:issue:`4000`) + +* :class:`~scrapy.http.Request` no longer requires a ``callback`` parameter + when an ``errback`` parameter is specified (:issue:`3586`, :issue:`4008`) + +* :class:`~scrapy.logformatter.LogFormatter` now supports some additional + methods: + + * :class:`~scrapy.logformatter.LogFormatter.download_error` for + download errors + + * :class:`~scrapy.logformatter.LogFormatter.item_error` for exceptions + raised during item processing by :ref:`item pipelines + ` + + * :class:`~scrapy.logformatter.LogFormatter.spider_error` for exceptions + raised from :ref:`spider callbacks ` + + (:issue:`374`, :issue:`3986`, :issue:`3989`, :issue:`4176`, :issue:`4188`) + +* The :setting:`FEED_URI` setting now supports :class:`pathlib.Path` values + (:issue:`3731`, :issue:`4074`) + +* A new :signal:`request_left_downloader` signal is sent when a request + leaves the downloader (:issue:`4303`) + +* Scrapy logs a warning when it detects a request callback or errback that + uses ``yield`` but also returns a value, since the returned value would be + lost (:issue:`3484`, :issue:`3869`) + +* :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError` + exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls` + attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`, + but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`) + +* :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use + ``super().__init__(**kwargs)`` instead of ``self._configure(kwargs)`` in + their ``__init__`` method, passing ``dont_fail=True`` to the parent + ``__init__`` method if needed, and accessing ``kwargs`` at ``self._kwargs`` + after calling their parent ``__init__`` method (:issue:`4193`, + :issue:`4370`) + +* A new ``keep_fragments`` parameter of + :func:`scrapy.utils.request.request_fingerprint` allows to generate + different fingerprints for requests with different fragments in their URL + (:issue:`4104`) + +* Download handlers (see :setting:`DOWNLOAD_HANDLERS`) may now use the + ``from_settings`` and ``from_crawler`` class methods that other Scrapy + components already supported (:issue:`4126`) + +* :class:`scrapy.utils.python.MutableChain.__iter__` now returns ``self``, + `allowing it to be used as a sequence `_ + (:issue:`4153`) + + +Bug fixes +~~~~~~~~~ + +* The :command:`crawl` command now also exits with exit code 1 when an + exception happens before the crawling starts (:issue:`4175`, :issue:`4207`) + +* :class:`LinkExtractor.extract_links + ` no longer + re-encodes the query string or URLs from non-UTF-8 responses in UTF-8 + (:issue:`998`, :issue:`1403`, :issue:`1949`, :issue:`4321`) + +* The first spider middleware (see :setting:`SPIDER_MIDDLEWARES`) now also + processes exceptions raised from callbacks that are generators + (:issue:`4260`, :issue:`4272`) + +* Redirects to URLs starting with 3 slashes (``///``) are now supported + (:issue:`4032`, :issue:`4042`) + +* :class:`~scrapy.http.Request` no longer accepts strings as ``url`` simply + because they have a colon (:issue:`2552`, :issue:`4094`) + +* The correct encoding is now used for attach names in + :class:`~scrapy.mail.MailSender` (:issue:`4229`, :issue:`4239`) + +* :class:`~scrapy.dupefilters.RFPDupeFilter`, the default + :setting:`DUPEFILTER_CLASS`, no longer writes an extra ``\r`` character on + each line in Windows, which made the size of the ``requests.seen`` file + unnecessarily large on that platform (:issue:`4283`) + +* Z shell auto-completion now looks for ``.html`` files, not ``.http`` files, + and covers the ``-h`` command-line switch (:issue:`4122`, :issue:`4291`) + +* Adding items to a :class:`scrapy.utils.datatypes.LocalCache` object + without a ``limit`` defined no longer raises a :exc:`TypeError` exception + (:issue:`4123`) + +* Fixed a typo in the message of the :exc:`ValueError` exception raised when + :func:`scrapy.utils.misc.create_instance` gets both ``settings`` and + ``crawler`` set to ``None`` (:issue:`4128`) + + +Documentation +~~~~~~~~~~~~~ + +* API documentation now links to an online, syntax-highlighted view of the + corresponding source code (:issue:`4148`) + +* Links to unexisting documentation pages now allow access to the sidebar + (:issue:`4152`, :issue:`4169`) + +* Cross-references within our documentation now display a tooltip when + hovered (:issue:`4173`, :issue:`4183`) + +* Improved the documentation about :meth:`LinkExtractor.extract_links + ` and + simplified :ref:`topics-link-extractors` (:issue:`4045`) + +* Clarified how :class:`ItemLoader.item ` + works (:issue:`3574`, :issue:`4099`) + +* Clarified that :func:`logging.basicConfig` should not be used when also + using :class:`~scrapy.crawler.CrawlerProcess` (:issue:`2149`, + :issue:`2352`, :issue:`3146`, :issue:`3960`) + +* Clarified the requirements for :class:`~scrapy.http.Request` objects + :ref:`when using persistence ` (:issue:`4124`, + :issue:`4139`) + +* Clarified how to install a :ref:`custom image pipeline + ` (:issue:`4034`, :issue:`4252`) + +* Fixed the signatures of the ``file_path`` method in :ref:`media pipeline + ` examples (:issue:`4290`) + +* Covered a backward-incompatible change in Scrapy 1.7.0 affecting custom + :class:`scrapy.core.scheduler.Scheduler` subclasses (:issue:`4274`) + +* Improved the ``README.rst`` and ``CODE_OF_CONDUCT.md`` files + (:issue:`4059`) + +* Documentation examples are now checked as part of our test suite and we + have fixed some of the issues detected (:issue:`4142`, :issue:`4146`, + :issue:`4171`, :issue:`4184`, :issue:`4190`) + +* Fixed logic issues, broken links and typos (:issue:`4247`, :issue:`4258`, + :issue:`4282`, :issue:`4288`, :issue:`4305`, :issue:`4308`, :issue:`4323`, + :issue:`4338`, :issue:`4359`, :issue:`4361`) + +* Improved consistency when referring to the ``__init__`` method of an object + (:issue:`4086`, :issue:`4088`) + +* Fixed an inconsistency between code and output in :ref:`intro-overview` + (:issue:`4213`) + +* Extended :mod:`~sphinx.ext.intersphinx` usage (:issue:`4147`, + :issue:`4172`, :issue:`4185`, :issue:`4194`, :issue:`4197`) + +* We now use a recent version of Python to build the documentation + (:issue:`4140`, :issue:`4249`) + +* Cleaned up documentation (:issue:`4143`, :issue:`4275`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Re-enabled proxy ``CONNECT`` tests (:issue:`2545`, :issue:`4114`) + +* Added Bandit_ security checks to our test suite (:issue:`4162`, + :issue:`4181`) + +* Added Flake8_ style checks to our test suite and applied many of the + corresponding changes (:issue:`3944`, :issue:`3945`, :issue:`4137`, + :issue:`4157`, :issue:`4167`, :issue:`4174`, :issue:`4186`, :issue:`4195`, + :issue:`4238`, :issue:`4246`, :issue:`4355`, :issue:`4360`, :issue:`4365`) + +* Improved test coverage (:issue:`4097`, :issue:`4218`, :issue:`4236`) + +* Started reporting slowest tests, and improved the performance of some of + them (:issue:`4163`, :issue:`4164`) + +* Fixed broken tests and refactored some tests (:issue:`4014`, :issue:`4095`, + :issue:`4244`, :issue:`4268`, :issue:`4372`) + +* Modified the :doc:`tox ` configuration to allow running tests + with any Python version, run Bandit_ and Flake8_ tests by default, and + enforce a minimum tox version programmatically (:issue:`4179`) + +* Cleaned up code (:issue:`3937`, :issue:`4208`, :issue:`4209`, + :issue:`4210`, :issue:`4212`, :issue:`4369`, :issue:`4376`, :issue:`4378`) + +.. _Bandit: https://bandit.readthedocs.io/ +.. _Flake8: https://flake8.pycqa.org/en/latest/ + + +.. _2-0-0-scheduler-queue-changes: + +Changes to scheduler queue classes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following changes may impact any custom queue classes of all types: + +* The ``push`` method no longer receives a second positional parameter + containing ``request.priority * -1``. If you need that value, get it + from the first positional parameter, ``request``, instead, or use + the new :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.priority` + method in :class:`scrapy.core.scheduler.ScrapyPriorityQueue` + subclasses. + +The following changes may impact custom priority queue classes: + +* In the ``__init__`` method or the ``from_crawler`` or ``from_settings`` + class methods: + + * The parameter that used to contain a factory function, + ``qfactory``, is now passed as a keyword parameter named + ``downstream_queue_cls``. + + * A new keyword parameter has been added: ``key``. It is a string + that is always an empty string for memory queues and indicates the + :setting:`JOB_DIR` value for disk queues. + + * The parameter for disk queues that contains data from the previous + crawl, ``startprios`` or ``slot_startprios``, is now passed as a + keyword parameter named ``startprios``. + + * The ``serialize`` parameter is no longer passed. The disk queue + class must take care of request serialization on its own before + writing to disk, using the + :func:`~scrapy.utils.reqser.request_to_dict` and + :func:`~scrapy.utils.reqser.request_from_dict` functions from the + :mod:`scrapy.utils.reqser` module. + +The following changes may impact custom disk and memory queue classes: + +* The signature of the ``__init__`` method is now + ``__init__(self, crawler, key)``. + +The following changes affect specifically the +:class:`~scrapy.core.scheduler.ScrapyPriorityQueue` and +:class:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue` classes from +:mod:`scrapy.core.scheduler` and may affect subclasses: + +* In the ``__init__`` method, most of the changes described above apply. + + ``__init__`` may still receive all parameters as positional parameters, + however: + + * ``downstream_queue_cls``, which replaced ``qfactory``, must be + instantiated differently. + + ``qfactory`` was instantiated with a priority value (integer). + + Instances of ``downstream_queue_cls`` should be created using + the new + :meth:`ScrapyPriorityQueue.qfactory ` + or + :meth:`DownloaderAwarePriorityQueue.pqfactory ` + methods. + + * The new ``key`` parameter displaced the ``startprios`` + parameter 1 position to the right. + +* The following class attributes have been added: + + * :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.crawler` + + * :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.downstream_queue_cls` + (details above) + + * :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.key` (details above) + +* The ``serialize`` attribute has been removed (details above) + +The following changes affect specifically the +:class:`~scrapy.core.scheduler.ScrapyPriorityQueue` class and may affect +subclasses: + +* A new :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.priority` + method has been added which, given a request, returns + ``request.priority * -1``. + + It is used in :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.push` + to make up for the removal of its ``priority`` parameter. + +* The ``spider`` attribute has been removed. Use + :attr:`crawler.spider ` + instead. + +The following changes affect specifically the +:class:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue` class and may +affect subclasses: + +* A new :attr:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue.pqueues` + attribute offers a mapping of downloader slot names to the + corresponding instances of + :attr:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue.downstream_queue_cls`. + +(:issue:`3884`) + + +.. _release-1.8.1: + +Scrapy 1.8.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + .. _release-1.8.0: @@ -171,6 +1717,8 @@ Deprecation removals * ``scrapy.xlib`` has been removed (:issue:`4015`) +.. _1.8-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -288,12 +1836,12 @@ Backward-incompatible changes :class:`~scrapy.http.Request` objects instead of arbitrary Python data structures. -* An additional ``crawler`` parameter has been added to the ``__init__`` method - of the :class:`scrapy.core.scheduler.Scheduler` class. - Custom scheduler subclasses which don't accept arbitrary parameters in - their ``__init__`` method might break because of this change. +* An additional ``crawler`` parameter has been added to the ``__init__`` + method of the :class:`~scrapy.core.scheduler.Scheduler` class. Custom + scheduler subclasses which don't accept arbitrary parameters in their + ``__init__`` method might break because of this change. - For more information, refer to the documentation for the :setting:`SCHEDULER` setting. + For more information, see :setting:`SCHEDULER`. See also :ref:`1.7-deprecation-removals` below. @@ -304,7 +1852,7 @@ New features * A new scheduler priority queue, ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant - scheduling improvement on crawls targetting multiple web domains, at the + scheduling improvement on crawls targeting multiple web domains, at the cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) * A new :attr:`Request.cb_kwargs ` attribute @@ -527,6 +2075,8 @@ The following deprecated settings have also been removed (:issue:`3578`): * ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`) +.. _1.7-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -1166,7 +2716,7 @@ Bug fixes - Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`). - Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`). - Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``, - ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). + ``DOWNLOADER_MIDDLEWARES``, ``EXTENSIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). Documentation ~~~~~~~~~~~~~ @@ -1340,7 +2890,7 @@ Bug fixes - Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse ` (:issue:`2225`). - Fix for invalid JSON and XML files when spider yields no items (:issue:`872`). -- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). +- Implement ``flush()`` for ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). Refactoring ~~~~~~~~~~~ @@ -2203,7 +3753,7 @@ Scrapy 0.24.3 (2014-08-09) - adding some xpath tips to selectors docs (:commit:`2d103e0`) - fix tests to account for https://github.com/scrapy/w3lib/pull/23 (:commit:`f8d366a`) - get_func_args maximum recursion fix #728 (:commit:`81344ea`) -- Updated input/ouput processor example according to #560. (:commit:`f7c4ea8`) +- Updated input/output processor example according to #560. (:commit:`f7c4ea8`) - Fixed Python syntax in tutorial. (:commit:`db59ed9`) - Add test case for tunneling proxy (:commit:`f090260`) - Bugfix for leaking Proxy-Authorization header to remote host when using tunneling (:commit:`d8793af`) @@ -2865,7 +4415,7 @@ Scrapyd changes ~~~~~~~~~~~~~~~ - Scrapyd now uses one process per spider -- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default) +- It stores one log file per spider run, and rotate them keeping the latest 5 logs per spider (by default) - A minimal web ui was added, available at http://localhost:6800 by default - There is now a ``scrapy server`` command to start a Scrapyd server of the current project @@ -2901,7 +4451,7 @@ New features and improvements - Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195) - Support for overriding default request headers per spider (#181) - Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186) -- Splitted Debian package into two packages - the library and the service (#187) +- Split Debian package into two packages - the library and the service (#187) - Scrapy log refactoring (#188) - New extension for keeping persistent spider contexts among different runs (#203) - Added ``dont_redirect`` request.meta key for avoiding redirects (#233) @@ -2922,7 +4472,7 @@ API changes - ``url`` and ``body`` attributes of Request objects are now read-only (#230) - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) -- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) +- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) - Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself. - Changes to Scrapy Manager (now called "Crawler"): - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` @@ -3069,13 +4619,14 @@ First release of Scrapy. .. _resource: https://docs.python.org/2/library/resource.html .. _robots.txt: https://www.robotstxt.org/ .. _scrapely: https://github.com/scrapy/scrapely +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ .. _Twisted: https://twistedmatrix.com/trac/ -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _w3lib: https://github.com/scrapy/w3lib .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 .. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/ .. _Zsh: https://www.zsh.org/ +.. _zstandard: https://pypi.org/project/zstandard/ diff --git a/docs/requirements.txt b/docs/requirements.txt index 773b92cea..a0930ba1e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -Sphinx>=2.1 -sphinx-hoverxref -sphinx-notfound-page -sphinx_rtd_theme +Sphinx>=3.0 +sphinx-hoverxref>=0.2b1 +sphinx-notfound-page>=0.4 +sphinx-rtd-theme>=0.5.2 \ No newline at end of file diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 1c461a511..900b19c7a 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -4,8 +4,6 @@ Core API ======== -.. versionadded:: 0.15 - This section documents the Scrapy core API, and it's intended for developers of extensions and middlewares. @@ -31,7 +29,7 @@ how you :ref:`configure the downloader middlewares .. class:: Crawler(spidercls, settings) The Crawler object must be instantiated with a - :class:`scrapy.spiders.Spider` subclass and a + :class:`scrapy.Spider` subclass and a :class:`scrapy.settings.Settings` object. .. attribute:: settings @@ -91,7 +89,7 @@ how you :ref:`configure the downloader middlewares provided while constructing the crawler, and it is created after the arguments given in the :meth:`crawl` method. - .. method:: crawl(\*args, \**kwargs) + .. method:: crawl(*args, **kwargs) Starts the crawler by instantiating its spider class with the given ``args`` and ``kwargs`` arguments, while setting the execution engine in @@ -198,7 +196,7 @@ SpiderLoader API match the request's url against the domains of the spiders. :param request: queried request - :type request: :class:`~scrapy.http.Request` instance + :type request: :class:`~scrapy.Request` instance .. _topics-api-signals: diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index ae25dfa2f..0c3a7ed88 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -67,7 +67,7 @@ this: the :ref:`Scheduler ` and asks for possible next Requests to crawl. -9. The process repeats (from step 1) until there are no more requests from the +9. The process repeats (from step 3) until there are no more requests from the :ref:`Scheduler `. Components @@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The Scheduler receives requests from the engine and enqueues them for feeding -them later (also to the engine) when the engine requests them. +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine +requests them. .. _component-downloader: @@ -104,7 +105,7 @@ Spiders ------- Spiders are custom classes written by Scrapy users to parse responses and -extract items (aka scraped items) from them or additional requests to +extract :ref:`items ` from them or additional requests to follow. For more information see :ref:`topics-spiders`. .. _component-pipelines: diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst new file mode 100644 index 000000000..8712d4268 --- /dev/null +++ b/docs/topics/asyncio.rst @@ -0,0 +1,83 @@ +.. _using-asyncio: + +======= +asyncio +======= + +.. versionadded:: 2.0 + +Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the +asyncio reactor `, you may use :mod:`asyncio` and +:mod:`asyncio`-powered libraries in any :doc:`coroutine `. + +.. _install-asyncio: + +Installing the asyncio reactor +============================== + +To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to +``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``. + +If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to +install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` +reactor manually. You can do that using +:func:`~scrapy.utils.reactor.install_reactor`:: + + install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to +use it instead of the default asyncio event loop. + +.. _asyncio-await-dfd: + +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations: :class:`~asyncio.SelectorEventLoop` (default before Python +3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop` +(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the +event loop class needs to be changed. Scrapy since VERSION does this +automatically when you change the :setting:`TWISTED_REACTOR` setting or call +:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor +by other means or use an older Scrapy version you need to call the following +code before installing the reactor:: + + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +You can put this in the same function that installs the reactor, if you do that +yourself, or in some code that runs before the reactor is installed, e.g. +``settings.py``. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + +Awaiting on Deferreds +===================== + +When the asyncio reactor isn't installed, you can await on Deferreds in the +coroutines directly. When it is installed, this is not possible anymore, due to +specifics of the Scrapy coroutine integration (the coroutines are wrapped into +:class:`asyncio.Future` objects, not into +:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into +Futures. Scrapy provides two helpers for this: + +.. autofunction:: scrapy.utils.defer.deferred_to_future +.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future +.. tip:: If you need to use these functions in code that aims to be compatible + with lower versions of Scrapy that do not provide these functions, + down to Scrapy 2.0 (earlier versions do not support + :mod:`asyncio`), you can copy the implementation of these functions + into your own code. diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 4317019fc..8e6aae65c 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies. AUTOTHROTTLE_TARGET_CONCURRENCY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 1.1 - Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 99469ebf1..0643df6a6 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -4,8 +4,6 @@ Benchmarking ============ -.. versionadded:: 0.17 - Scrapy comes with a simple benchmarking suite that spawns a local HTTP server and crawls it at the maximum possible speed. The goal of this benchmarking is to get an idea of how Scrapy performs in your hardware, in order to have a @@ -83,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -In the future, more cases will be added to the benchmarking suite to cover -other common scenarios. +Use scrapy-bench_ for more complex benchmarking. + +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench \ No newline at end of file diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index a0dcba90d..eef6b36ff 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,8 +6,6 @@ Command line tool ================= -.. versionadded:: 0.10 - Scrapy is controlled through the ``scrapy`` command-line tool, to be referred here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -468,7 +466,7 @@ Supported options: * ``--callback`` or ``-c``: spider method to use as callback for parsing the response -* ``--meta`` or ``-m``: additional request meta that will be passed to the callback +* ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' * ``--cbkwargs``: additional keyword arguments that will be passed to the callback. @@ -491,6 +489,10 @@ Supported options: * ``--verbose`` or ``-v``: display information for each depth level +* ``--output`` or ``-o``: dump scraped items to a file + + .. versionadded:: 2.3 + .. skip: start Usage example:: @@ -562,8 +564,6 @@ and Platform info, which is useful for bug reports. bench ----- -.. versionadded:: 0.17 - * Syntax: ``scrapy bench`` * Requires project: *no* @@ -598,8 +598,6 @@ Example: Register commands via setup.py entry points ------------------------------------------- -.. note:: This is an experimental feature, use with caution. - You can also add Scrapy commands from an external library by adding a ``scrapy.commands`` section in the entry points of the library ``setup.py`` file. diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 43db8f101..ef296dc9e 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -4,8 +4,6 @@ Spiders Contracts ================= -.. versionadded:: 0.15 - Testing spiders can get particularly annoying and while nothing prevents you from writing unit tests the task gets cumbersome quickly. Scrapy offers an integrated way of testing your spiders by the means of contracts. @@ -39,7 +37,7 @@ This callback is tested using three built-in contracts: .. class:: CallbackKeywordArgumentsContract - This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` + This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` attribute for the sample request. It must be a valid JSON dictionary. :: @@ -78,10 +76,10 @@ override three methods: .. module:: scrapy.contracts -.. class:: Contract(method, \*args) +.. class:: Contract(method, *args) :param method: callback function to which the contract is associated - :type method: function + :type method: collections.abc.Callable :param args: list of arguments passed into the docstring (whitespace separated) @@ -90,7 +88,7 @@ override three methods: .. method:: Contract.adjust_request_args(args) This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.http.Request` is used by default, + for request object. :class:`~scrapy.Request` is used by default, but this can be changed with the ``request_cls`` attribute. If multiple contracts in chain have this attribute defined, the last one is used. @@ -136,7 +134,7 @@ Detecting check runs ==================== When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is -set to the ``true`` string. You can use `os.environ`_ to perform any change to +set to the ``true`` string. You can use :data:`os.environ` to perform any change to your spiders or your settings when ``scrapy check`` is used:: import os @@ -148,5 +146,3 @@ your spiders or your settings when ``scrapy check`` is used:: def __init__(self): if os.environ.get('SCRAPY_CHECK'): pass # Do some scraper adjustments when a check is running - -.. _os.environ: https://docs.python.org/3/library/os.html#os.environ diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst new file mode 100644 index 000000000..2aef755c7 --- /dev/null +++ b/docs/topics/coroutines.rst @@ -0,0 +1,110 @@ +========== +Coroutines +========== + +.. versionadded:: 2.0 + +Scrapy has :ref:`partial support ` for the +:ref:`coroutine syntax `. + +.. _coroutine-support: + +Supported callables +=================== + +The following callables may be defined as coroutines using ``async def``, and +hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): + +- :class:`~scrapy.Request` callbacks. + + .. note:: The callback output is not processed until the whole callback + finishes. + + As a side effect, if the callback raises an exception, none of its + output is processed. + + This is a known caveat of the current implementation that we aim to + address in a future version of Scrapy. + +- The :meth:`process_item` method of + :ref:`item pipelines `. + +- The + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`, + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`, + and + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` + methods of + :ref:`downloader middlewares `. + +- :ref:`Signal handlers that support deferreds `. + +Usage +===== + +There are several use cases for coroutines in Scrapy. Code that would +return Deferreds when written for previous Scrapy versions, such as downloader +middlewares and signal handlers, can be rewritten to be shorter and cleaner:: + + from itemadapter import ItemAdapter + + class DbPipeline: + def _update_item(self, data, item): + adapter = ItemAdapter(item) + adapter['field'] = data + return item + + def process_item(self, item, spider): + adapter = ItemAdapter(item) + dfd = db.get_some_data(adapter['id']) + dfd.addCallback(self._update_item, item) + return dfd + +becomes:: + + from itemadapter import ItemAdapter + + class DbPipeline: + async def process_item(self, item, spider): + adapter = ItemAdapter(item) + adapter['field'] = await db.get_some_data(adapter['id']) + return item + +Coroutines may be used to call asynchronous code. This includes other +coroutines, functions that return Deferreds and functions that return +:term:`awaitable objects ` such as :class:`~asyncio.Future`. +This means you can use many useful Python libraries providing such code:: + + class MySpiderDeferred(Spider): + # ... + async def parse(self, response): + additional_response = await treq.get('https://additional.url') + additional_data = await treq.content(additional_response) + # ... use response and additional_data to yield items and requests + + class MySpiderAsyncio(Spider): + # ... + async def parse(self, response): + async with aiohttp.ClientSession() as session: + async with session.get('https://additional.url') as additional_response: + additional_data = await additional_response.text() + # ... use response and additional_data to yield items and requests + +.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the + :mod:`asyncio` loop and to use them you need to + :doc:`enable asyncio support in Scrapy`. + +.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor, + you need to :ref:`wrap them`. + +Common use cases for asynchronous code include: + +* requesting data from websites, databases and other services (in callbacks, + pipelines and middlewares); +* storing data in databases (in pipelines and middlewares); +* delaying the spider initialization until some external event (in the + :signal:`spider_opened` handler); +* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see + :ref:`the screenshot pipeline example`). + +.. _aio-libs: https://github.com/aio-libs diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index d75f17301..4d452b4df 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -36,7 +36,7 @@ Consider the following Scrapy spider below:: Basically this is a simple spider which parses two pages of items (the start_urls). Items also have a details page with additional information, so we -use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a +use the ``cb_kwargs`` functionality of :class:`~scrapy.Request` to pass a partially populated item. diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index 361914a29..961d6dc01 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -14,7 +14,7 @@ spiders come in. Popular choices for deploying Scrapy spiders are: * :ref:`Scrapyd ` (open source) -* :ref:`Scrapy Cloud ` (cloud-based) +* :ref:`Zyte Scrapy Cloud ` (cloud-based) .. _deploy-scrapyd: @@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers. .. _deploy-scrapy-cloud: -Deploying to Scrapy Cloud -========================= +Deploying to Zyte Scrapy Cloud +============================== -`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_, -the company behind Scrapy. +`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company +behind Scrapy. -Scrapy Cloud removes the need to setup and monitor servers -and provides a nice UI to manage spiders and review scraped items, -logs and stats. +Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a +nice UI to manage spiders and review scraped items, logs and stats. -To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool. -Please refer to the `Scrapy Cloud documentation`_ for more information. +To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line +tool. +Please refer to the `Zyte Scrapy Cloud documentation`_ for more information. -Scrapy Cloud is compatible with Scrapyd and one can switch between +Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between them as needed - the configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``. -.. _Scrapyd: https://github.com/scrapy/scrapyd .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud +.. _Scrapyd: https://github.com/scrapy/scrapyd .. _scrapyd-client: https://github.com/scrapy/scrapyd-client -.. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html -.. _Scrapinghub: https://scrapinghub.com/ +.. _shub: https://shub.readthedocs.io/en/latest/ +.. _Zyte: https://zyte.com/ +.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/ +.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index f1b0964c6..057b1ec62 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping ================================================= Here is a general guide on how to use your browser's Developer Tools -to ease the scraping process. Today almost all browsers come with +to ease the scraping process. Today almost all browsers come with built in `Developer Tools`_ and although we will use Firefox in this -guide, the concepts are applicable to any other browser. +guide, the concepts are applicable to any other browser. In this guide we'll introduce the basic tools to use from a browser's Developer Tools by scraping `quotes.toscrape.com`_. @@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things: Inspecting a website ==================== -By far the most handy feature of the Developer Tools is the `Inspector` -feature, which allows you to inspect the underlying HTML code of -any webpage. To demonstrate the Inspector, let's look at the +By far the most handy feature of the Developer Tools is the `Inspector` +feature, which allows you to inspect the underlying HTML code of +any webpage. To demonstrate the Inspector, let's look at the `quotes.toscrape.com`_-site. On the site we have a total of ten quotes from various authors with specific -tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes -on this page, without any meta-information about authors, tags, etc. +tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes +on this page, without any meta-information about authors, tags, etc. -Instead of viewing the whole source code for the page, we can simply right click +Instead of viewing the whole source code for the page, we can simply right click on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`. In it you should see something like this: @@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] -Adding ``text()`` at the end we are able to extract the first quote with this +Adding ``text()`` at the end we are able to extract the first quote with this basic selector. But this XPath is not really that clever. All it does is -go down a desired path in the source code starting from ``html``. So let's -see if we can refine our XPath a bit: +go down a desired path in the source code starting from ``html``. So let's +see if we can refine our XPath a bit: -If we check the `Inspector` again we'll see that directly beneath our -expanded ``div`` tag we have nine identical ``div`` tags, each with the -same attributes as our first. If we expand any of them, we'll see the same +If we check the `Inspector` again we'll see that directly beneath our +expanded ``div`` tag we have nine identical ``div`` tags, each with the +same attributes as our first. If we expand any of them, we'll see the same structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can -expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and +expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and see each quote: .. code-block:: html @@ -121,7 +121,7 @@ see each quote: With this knowledge we can refine our XPath: Instead of a path to follow, -we'll simply select all ``span`` tags with the ``class="text"`` by using +we'll simply select all ``span`` tags with the ``class="text"`` by using the `has-class-extension`_: >>> response.xpath('//span[has-class("text")]/text()').getall() @@ -130,45 +130,45 @@ the `has-class-extension`_: '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', ...] -And with one simple, cleverer XPath we are able to extract all quotes from -the page. We could have constructed a loop over our first XPath to increase -the number of the last ``div``, but this would have been unnecessarily +And with one simple, cleverer XPath we are able to extract all quotes from +the page. We could have constructed a loop over our first XPath to increase +the number of the last ``div``, but this would have been unnecessarily complex and by simply constructing an XPath with ``has-class("text")`` -we were able to extract all quotes in one line. +we were able to extract all quotes in one line. -The `Inspector` has a lot of other helpful features, such as searching in the +The `Inspector` has a lot of other helpful features, such as searching in the source code or directly scrolling to an element you selected. Let's demonstrate -a use case: +a use case: -Say you want to find the ``Next`` button on the page. Type ``Next`` into the -search bar on the top right of the `Inspector`. You should get two results. -The first is a ``li`` tag with the ``class="next"``, the second the text +Say you want to find the ``Next`` button on the page. Type ``Next`` into the +search bar on the top right of the `Inspector`. You should get two results. +The first is a ``li`` tag with the ``class="next"``, the second the text of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``. If you hover over the tag, you'll see the button highlighted. From here -we could easily create a :ref:`Link Extractor ` to -follow the pagination. On a simple site such as this, there may not be +we could easily create a :ref:`Link Extractor ` to +follow the pagination. On a simple site such as this, there may not be the need to find an element visually but the ``Scroll into View`` function -can be quite useful on complex sites. +can be quite useful on complex sites. Note that the search bar can also be used to search for and test CSS -selectors. For example, you could search for ``span.text`` to find -all quote texts. Instead of a full text search, this searches for -exactly the ``span`` tag with the ``class="text"`` in the page. +selectors. For example, you could search for ``span.text`` to find +all quote texts. Instead of a full text search, this searches for +exactly the ``span`` tag with the ``class="text"`` in the page. .. _topics-network-tool: The Network-tool ================ While scraping you may come across dynamic webpages where some parts -of the page are loaded dynamically through multiple requests. While -this can be quite tricky, the `Network`-tool in the Developer Tools +of the page are loaded dynamically through multiple requests. While +this can be quite tricky, the `Network`-tool in the Developer Tools greatly facilitates this task. To demonstrate the Network-tool, let's -take a look at the page `quotes.toscrape.com/scroll`_. +take a look at the page `quotes.toscrape.com/scroll`_. -The page is quite similar to the basic `quotes.toscrape.com`_-page, -but instead of the above-mentioned ``Next`` button, the page -automatically loads new quotes when you scroll to the bottom. We -could go ahead and try out different XPaths directly, but instead +The page is quite similar to the basic `quotes.toscrape.com`_-page, +but instead of the above-mentioned ``Next`` button, the page +automatically loads new quotes when you scroll to the bottom. We +could go ahead and try out different XPaths directly, but instead we'll check another quite useful command from the Scrapy shell: .. skip: next @@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell: (...) >>> view(response) -A browser window should open with the webpage but with one -crucial difference: Instead of the quotes we just see a greenish -bar with the word ``Loading...``. +A browser window should open with the webpage but with one +crucial difference: Instead of the quotes we just see a greenish +bar with the word ``Loading...``. .. image:: _images/network_01.png :width: 777 @@ -189,21 +189,21 @@ bar with the word ``Loading...``. :alt: Response from quotes.toscrape.com/scroll The ``view(response)`` command let's us view the response our -shell or later our spider receives from the server. Here we see -that some basic template is loaded which includes the title, +shell or later our spider receives from the server. Here we see +that some basic template is loaded which includes the title, the login-button and the footer, but the quotes are missing. This tells us that the quotes are being loaded from a different request -than ``quotes.toscrape/scroll``. +than ``quotes.toscrape/scroll``. -If you click on the ``Network`` tab, you will probably only see -two entries. The first thing we do is enable persistent logs by -clicking on ``Persist Logs``. If this option is disabled, the +If you click on the ``Network`` tab, you will probably only see +two entries. The first thing we do is enable persistent logs by +clicking on ``Persist Logs``. If this option is disabled, the log is automatically cleared each time you navigate to a different -page. Enabling this option is a good default, since it gives us -control on when to clear the logs. +page. Enabling this option is a good default, since it gives us +control on when to clear the logs. If we reload the page now, you'll see the log get populated with six -new requests. +new requests. .. image:: _images/network_02.png :width: 777 @@ -212,31 +212,31 @@ new requests. Here we see every request that has been made when reloading the page and can inspect each request and its response. So let's find out -where our quotes are coming from: +where our quotes are coming from: -First click on the request with the name ``scroll``. On the right +First click on the request with the name ``scroll``. On the right you can now inspect the request. In ``Headers`` you'll find details about the request headers, such as the URL, the method, the IP-address, and so on. We'll ignore the other tabs and click directly on ``Response``. -What you should see in the ``Preview`` pane is the rendered HTML-code, -that is exactly what we saw when we called ``view(response)`` in the -shell. Accordingly the ``type`` of the request in the log is ``html``. -The other requests have types like ``css`` or ``js``, but what -interests us is the one request called ``quotes?page=1`` with the -type ``json``. +What you should see in the ``Preview`` pane is the rendered HTML-code, +that is exactly what we saw when we called ``view(response)`` in the +shell. Accordingly the ``type`` of the request in the log is ``html``. +The other requests have types like ``css`` or ``js``, but what +interests us is the one request called ``quotes?page=1`` with the +type ``json``. -If we click on this request, we see that the request URL is +If we click on this request, we see that the request URL is ``http://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click -on the request and open ``Open in new tab`` to get a better overview. +on the request and open ``Open in new tab`` to get a better overview. .. image:: _images/network_03.png :width: 777 :height: 375 :alt: JSON-object returned from the quotes.toscrape API -With this response we can now easily parse the JSON-object and +With this response we can now easily parse the JSON-object and also request each page to get every quote on the site:: import scrapy @@ -255,17 +255,17 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page) + url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) -This spider starts at the first page of the quotes-API. With each -response, we parse the ``response.text`` and assign it to ``data``. -This lets us operate on the JSON-object like on a Python dictionary. +This spider starts at the first page of the quotes-API. With each +response, we parse the ``response.text`` and assign it to ``data``. +This lets us operate on the JSON-object like on a Python dictionary. We iterate through the ``quotes`` and print out the ``quote["text"]``. -If the handy ``has_next`` element is ``true`` (try loading +If the handy ``has_next`` element is ``true`` (try loading `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a -page-number greater than 10), we increment the ``page`` attribute -and ``yield`` a new request, inserting the incremented page-number +page-number greater than 10), we increment the ``page`` attribute +and ``yield`` a new request, inserting the incremented page-number into our ``url``. .. _requests-from-curl: @@ -274,7 +274,7 @@ In more complex websites, it could be difficult to easily reproduce the requests, as we could need to add ``headers`` or ``cookies`` to make it work. In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the -:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent +:meth:`~scrapy.Request.from_curl()` method to generate an equivalent request:: from scrapy import Request @@ -289,11 +289,16 @@ request:: "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") Alternatively, if you want to know the arguments needed to recreate that -request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs` -function to get a dictionary with the equivalent arguments. +request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs` +function to get a dictionary with the equivalent arguments: + +.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs + +Note that to translate a cURL command into a Scrapy request, +you may use `curl2scrapy `_. As you can see, with a few inspections in the `Network`-tool we -were able to easily replicate the dynamic requests of the scrolling +were able to easily replicate the dynamic requests of the scrolling functionality of the page. Crawling dynamic pages can be quite daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0297ef3a0..caf44a903 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -76,7 +76,7 @@ object gives you access, for example, to the :ref:`settings `. middleware. :meth:`process_request` should either: return ``None``, return a - :class:`~scrapy.http.Response` object, return a :class:`~scrapy.http.Request` + :class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request` object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`. If it returns ``None``, Scrapy will continue processing this request, executing all @@ -88,7 +88,7 @@ object gives you access, for example, to the :ref:`settings `. or the appropriate download function; it'll return that response. The :meth:`process_response` methods of installed middleware is always called on every response. - If it returns a :class:`~scrapy.http.Request` object, Scrapy will stop calling + If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling process_request methods and reschedule the returned request. Once the newly returned request is performed, the appropriate middleware chain will be called on the downloaded response. @@ -100,22 +100,22 @@ object gives you access, for example, to the :ref:`settings `. ignored and not logged (unlike other exceptions). :param request: the request being processed - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_response(request, response, spider) :meth:`process_response` should either: return a :class:`~scrapy.http.Response` - object, return a :class:`~scrapy.http.Request` object or + object, return a :class:`~scrapy.Request` object or raise a :exc:`~scrapy.exceptions.IgnoreRequest` exception. If it returns a :class:`~scrapy.http.Response` (it could be the same given response, or a brand-new one), that response will continue to be processed with the :meth:`process_response` of the next middleware in the chain. - If it returns a :class:`~scrapy.http.Request` object, the middleware chain is + If it returns a :class:`~scrapy.Request` object, the middleware chain is halted and the returned request is rescheduled to be downloaded in the future. This is the same behavior as if a request is returned from :meth:`process_request`. @@ -124,13 +124,13 @@ object gives you access, for example, to the :ref:`settings `. exception, it is ignored and not logged (unlike other exceptions). :param request: the request that originated the response - :type request: is a :class:`~scrapy.http.Request` object + :type request: is a :class:`~scrapy.Request` object :param response: the response being processed :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_exception(request, exception, spider) @@ -139,7 +139,7 @@ object gives you access, for example, to the :ref:`settings `. exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception) :meth:`process_exception` should return: either ``None``, - a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.http.Request` object. + a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_exception` methods of installed middleware, @@ -149,19 +149,19 @@ object gives you access, for example, to the :ref:`settings `. method chain of installed middleware is started, and Scrapy won't bother calling any other :meth:`process_exception` methods of middleware. - If it returns a :class:`~scrapy.http.Request` object, the returned request is + If it returns a :class:`~scrapy.Request` object, the returned request is rescheduled to be downloaded in the future. This stops the execution of :meth:`process_exception` methods of the middleware the same as returning a response would. :param request: the request that generated the exception - :type request: is a :class:`~scrapy.http.Request` object + :type request: is a :class:`~scrapy.Request` object :param exception: the raised exception :type exception: an ``Exception`` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) @@ -202,6 +202,16 @@ CookiesMiddleware sends them back on subsequent requests (from that spider), just like web browsers do. + .. caution:: When non-UTF8 encoded byte sequences are passed to a + :class:`~scrapy.Request`, the ``CookiesMiddleware`` will log + a warning. Refer to :ref:`topics-logging-advanced-customization` + to customize the logging behaviour. + + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + The following settings can be used to configure the cookie middleware: * :setting:`COOKIES_ENABLED` @@ -212,8 +222,6 @@ The following settings can be used to configure the cookie middleware: Multiple cookie sessions per spider ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 0.15 - There is support for keeping multiple cookie sessions per spider by using the :reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar (session), but you can pass an identifier to use different ones. @@ -250,7 +258,7 @@ web server and received cookies in :class:`~scrapy.http.Response` will **not** be merged with the existing cookies. For more detailed information see the ``cookies`` parameter in -:class:`~scrapy.http.Request`. +:class:`~scrapy.Request`. .. setting:: COOKIES_DEBUG @@ -315,8 +323,21 @@ HttpAuthMiddleware This middleware authenticates all requests generated from certain spiders using `Basic access authentication`_ (aka. HTTP auth). - To enable HTTP authentication from certain spiders, set the ``http_user`` - and ``http_pass`` attributes of those spiders. + To enable HTTP authentication for a spider, set the ``http_user`` and + ``http_pass`` spider attributes to the authentication data and the + ``http_auth_domain`` spider attribute to the domain which requires this + authentication (its subdomains will be also handled in the same way). + You can set ``http_auth_domain`` to ``None`` to enable the + authentication for all requests but you risk leaking your authentication + credentials to unrelated domains. + + .. warning:: + In previous Scrapy versions HttpAuthMiddleware sent the authentication + data with all requests, which is a security problem if the spider + makes requests to several different domains. Currently if the + ``http_auth_domain`` attribute is not set, the middleware will use the + domain of the first request, which will work for some spiders but not + for others. In the future the middleware will produce an error instead. Example:: @@ -326,6 +347,7 @@ HttpAuthMiddleware http_user = 'someuser' http_pass = 'somepass' + http_auth_domain = 'intranet.example.com' name = 'intranet.example.com' # .. rest of the spider code omitted ... @@ -470,8 +492,6 @@ DBM storage backend .. class:: DbmCacheStorage - .. versionadded:: 0.13 - A DBM_ storage backend is also available for the HTTP cache middleware. By default, it uses the :mod:`dbm`, but you can change it with the @@ -495,7 +515,7 @@ defines the methods described below. the :signal:`open_spider ` signal. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: close_spider(spider) @@ -503,27 +523,27 @@ defines the methods described below. the :signal:`close_spider ` signal. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: retrieve_response(spider, request) Return response if present in cache, or ``None`` otherwise. :param spider: the spider which generated the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param request: the request to find cached response for - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object .. method:: store_response(spider, request, response) Store the given response in the cache. :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param request: the corresponding request the spider generated - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param response: the response to store in the cache :type response: :class:`~scrapy.http.Response` object @@ -544,15 +564,10 @@ settings: HTTPCACHE_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.11 - Default: ``False`` Whether the HTTP cache will be enabled. -.. versionchanged:: 0.11 - Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache. - .. setting:: HTTPCACHE_EXPIRATION_SECS HTTPCACHE_EXPIRATION_SECS @@ -565,9 +580,6 @@ Expiration time for cached requests, in seconds. Cached requests older than this time will be re-downloaded. If zero, cached requests will never expire. -.. versionchanged:: 0.11 - Before 0.11, zero meant cached requests always expire. - .. setting:: HTTPCACHE_DIR HTTPCACHE_DIR @@ -584,8 +596,6 @@ project data dir. For more info see: :ref:`topics-project-structure`. HTTPCACHE_IGNORE_HTTP_CODES ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``[]`` Don't cache response with these HTTP codes. @@ -604,8 +614,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade HTTPCACHE_IGNORE_SCHEMES ^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``['file']`` Don't cache responses with these URI schemes. @@ -624,8 +632,6 @@ The class which implements the cache storage backend. HTTPCACHE_DBM_MODULE ^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``'dbm'`` The database module to use in the :ref:`DBM storage backend @@ -636,8 +642,6 @@ The database module to use in the :ref:`DBM storage backend HTTPCACHE_POLICY ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.18 - Default: ``'scrapy.extensions.httpcache.DummyPolicy'`` The class which implements the cache policy. @@ -647,8 +651,6 @@ The class which implements the cache policy. HTTPCACHE_GZIP ^^^^^^^^^^^^^^ -.. versionadded:: 1.0 - Default: ``False`` If enabled, will compress all cached data with gzip. @@ -659,8 +661,6 @@ This setting is specific to the Filesystem backend. HTTPCACHE_ALWAYS_STORE ^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``False`` If enabled, will cache pages unconditionally. @@ -679,8 +679,6 @@ responses you feed to the cache middleware. HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``[]`` List of Cache-Control directives in responses to be ignored. @@ -705,11 +703,14 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ responses, - provided `brotlipy`_ is installed. + This middleware also supports decoding `brotli-compressed`_ as well as + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is + installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ +.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt +.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -730,16 +731,14 @@ HttpProxyMiddleware .. module:: scrapy.downloadermiddlewares.httpproxy :synopsis: Http Proxy Middleware -.. versionadded:: 0.8 - .. reqmeta:: proxy .. class:: HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the - ``proxy`` meta value for :class:`~scrapy.http.Request` objects. + ``proxy`` meta value for :class:`~scrapy.Request` objects. - Like the Python standard library modules `urllib`_ and `urllib2`_, it obeys + Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: * ``http_proxy`` @@ -751,9 +750,6 @@ HttpProxyMiddleware Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. -.. _urllib: https://docs.python.org/2/library/urllib.html -.. _urllib2: https://docs.python.org/2/library/urllib2.html - RedirectMiddleware ------------------ @@ -767,12 +763,12 @@ RedirectMiddleware .. reqmeta:: redirect_urls The urls which the request goes through (while being redirected) can be found -in the ``redirect_urls`` :attr:`Request.meta ` key. +in the ``redirect_urls`` :attr:`Request.meta ` key. .. reqmeta:: redirect_reasons The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the -``redirect_reasons`` :attr:`Request.meta ` key. For +``redirect_reasons`` :attr:`Request.meta ` key. For example: ``[301, 302, 307, 'meta refresh']``. The format of a reason depends on the middleware that handled the corresponding @@ -788,7 +784,7 @@ settings (see the settings documentation for more info): .. reqmeta:: dont_redirect -If :attr:`Request.meta ` has ``dont_redirect`` +If :attr:`Request.meta ` has ``dont_redirect`` key set to True, the request will be ignored by this middleware. If you want to handle some redirect status codes in your spider, you can @@ -801,7 +797,7 @@ responses (and pass them through to your spider) you can do this:: handle_httpstatus_list = [301, 302] The ``handle_httpstatus_list`` key of :attr:`Request.meta -` can also be used to specify which response codes to +` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` to ``True`` if you want to allow any response code for a request. @@ -815,8 +811,6 @@ RedirectMiddleware settings REDIRECT_ENABLED ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Redirect middleware will be enabled. @@ -829,6 +823,7 @@ REDIRECT_MAX_TIMES Default: ``20`` The maximum number of redirections that will be followed for a single request. +After this maximum, the request's response is returned as is. MetaRefreshMiddleware --------------------- @@ -857,8 +852,6 @@ MetaRefreshMiddleware settings METAREFRESH_ENABLED ^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.17 - Default: ``True`` Whether the Meta Refresh middleware will be enabled. @@ -872,6 +865,10 @@ Default: ``[]`` Meta tags within these tags are ignored. +.. versionchanged:: 2.0 + The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from + ``['script', 'noscript']`` to ``[]``. + .. setting:: METAREFRESH_MAXDELAY METAREFRESH_MAXDELAY @@ -906,9 +903,14 @@ settings (see the settings documentation for more info): .. reqmeta:: dont_retry -If :attr:`Request.meta ` has ``dont_retry`` key +If :attr:`Request.meta ` has ``dont_retry`` key set to True, the request will be ignored by this middleware. +To retry requests from a spider callback, you can use the +:func:`get_retry_request` function: + +.. autofunction:: get_retry_request + RetryMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -917,8 +919,6 @@ RetryMiddleware Settings RETRY_ENABLED ^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Retry middleware will be enabled. @@ -933,7 +933,7 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. Maximum number of retries can also be specified per-request using -:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. +:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. When initialized, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. @@ -951,6 +951,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_PRIORITY_ADJUST + +RETRY_PRIORITY_ADJUST +--------------------- + +Default: ``-1`` + +Adjust retry request priority relative to original request: + +- a positive priority adjust means higher priority. +- **a negative priority adjust (default) means lower priority.** + .. _topics-dlmw-robots: @@ -988,7 +1000,7 @@ RobotsTxtMiddleware .. reqmeta:: dont_obey_robotstxt -If :attr:`Request.meta ` has +If :attr:`Request.meta ` has ``dont_obey_robotstxt`` key set to True the request will be ignored by this middleware even if :setting:`ROBOTSTXT_OBEY` is enabled. @@ -1032,8 +1044,7 @@ Scrapy uses this parser by default. RobotFileParser ~~~~~~~~~~~~~~~ -Based on `RobotFileParser -`_: +Based on :class:`~urllib.robotparser.RobotFileParser`: * is Python's built-in robots.txt_ parser @@ -1073,9 +1084,13 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` + .. warning:: `Upstream issue #122 + `_ prevents reppy usage in Python 3.9+. + * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` + .. _rerp-parser: Robotexclusionrulesparser @@ -1173,8 +1188,6 @@ AjaxCrawlMiddleware Settings AJAXCRAWL_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.21 - Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. You may want to diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index b98133676..ea5d06210 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -62,9 +62,9 @@ download the webpage with an HTTP client like curl_ or wget_ and see if the information can be found in the response they get. If they get a response with the desired data, modify your Scrapy -:class:`~scrapy.http.Request` to match that of the other HTTP client. For +:class:`~scrapy.Request` to match that of the other HTTP client. For example, try using the same user-agent string (:setting:`USER_AGENT`) or the -same :attr:`~scrapy.http.Request.headers`. +same :attr:`~scrapy.Request.headers`. If they also get a response without the desired data, you’ll need to take steps to make your request more similar to that of the web browser. See @@ -81,14 +81,14 @@ Use the :ref:`network tool ` of your web browser to see how your web browser performs the desired request, and try to reproduce that request with Scrapy. -It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP +It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.http.FormRequest`) of that request. +form parameters (see :class:`~scrapy.FormRequest`) of that request. As all major browsers allow to export the requests in `cURL `_ format, Scrapy incorporates the method -:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent -:class:`~scrapy.http.Request` from a cURL command. To get more information +:meth:`~scrapy.Request.from_curl()` to generate an equivalent +:class:`~scrapy.Request` from a cURL command. To get more information visit :ref:`request from curl ` inside the network tool section. @@ -104,6 +104,9 @@ If you get the expected response `sometimes`, but not always, the issue is probably not your request, but the target server. The target server might be buggy, overloaded, or :ref:`banning ` some of your requests. +Note that to translate a cURL command into a Scrapy request, +you may use `curl2scrapy `_. + .. _topics-handling-response-formats: Handling different response formats @@ -115,14 +118,14 @@ data from it depends on the type of response: - If the response is HTML or XML, use :ref:`selectors ` as usual. -- If the response is JSON, use `json.loads`_ to load the desired data from +- If the response is JSON, use :func:`json.loads` to load the desired data from :attr:`response.text `:: data = json.loads(response.text) If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a - :class:`~scrapy.selector.Selector` and then + :class:`~scrapy.Selector` and then :ref:`use it ` as usual:: selector = Selector(data['html']) @@ -130,8 +133,9 @@ data from it depends on the type of response: - If the response is JavaScript, or HTML with a ``