diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f347a0cd0..d9e4a2831 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.0.0 +current_version = 2.5.0 commit = True tag = True tag_name = {new_version} diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..1c503fb0b --- /dev/null +++ b/.flake8 @@ -0,0 +1,19 @@ +[flake8] + +max-line-length = 119 +ignore = W503 + +exclude = +# Exclude files that are meant to provide top-level imports +# E402: Module level import not at top of file +# F401: Module imported but unused + scrapy/__init__.py E402 + scrapy/core/downloader/handlers/http.py F401 + scrapy/http/__init__.py F401 + scrapy/linkextractors/__init__.py E402 F401 + scrapy/selector/__init__.py F401 + scrapy/spiders/__init__.py E402 F401 + + # Issues pending a review: + scrapy/utils/url.py F403 F405 + tests/test_loader.py E741 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfbdf4208 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/sample_data/** binary diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 000000000..02c647da9 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,40 @@ +name: Checks +on: [push, pull_request] + +jobs: + checks: + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 3.9 + env: + TOXENV: security + - python-version: 3.9 + env: + TOXENV: flake8 + # Pylint requires installing reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 + - python-version: 3.8 + env: + TOXENV: pylint + - python-version: 3.9 + env: + TOXENV: typing + - python-version: 3.8 # Keep in sync with .readthedocs.yml + env: + TOXENV: docs + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run check + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..b48066ea4 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,31 @@ +name: Publish +on: [push] + +jobs: + publish: + runs-on: ubuntu-18.04 + if: startsWith(github.event.ref, 'refs/tags/') + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Check Tag + id: check-release-tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then + echo ::set-output name=release_tag::true + fi + + - name: Publish to PyPI + if: steps.check-release-tag.outputs.release_tag == 'true' + run: | + pip install --upgrade setuptools wheel twine + python setup.py sdist bdist_wheel + export TWINE_USERNAME=__token__ + export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} + twine upload dist/* diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml new file mode 100644 index 000000000..4f8f7a19d --- /dev/null +++ b/.github/workflows/tests-macos.yml @@ -0,0 +1,25 @@ +name: macOS +on: [push, pull_request] + +jobs: + tests: + runs-on: macos-10.15 + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + run: | + pip install -U tox + tox -e py + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml new file mode 100644 index 000000000..df5ee9d69 --- /dev/null +++ b/.github/workflows/tests-ubuntu.yml @@ -0,0 +1,74 @@ +name: Ubuntu +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + - python-version: 3.9 + env: + TOXENV: py + - python-version: pypy3 + env: + TOXENV: pypy3 + PYPY_VERSION: 3.6-v7.3.1 + + # pinned deps + - python-version: 3.6.12 + env: + TOXENV: pinned + - python-version: 3.6.12 + env: + TOXENV: asyncio-pinned + - python-version: pypy3 + env: + TOXENV: pypy3-pinned + PYPY_VERSION: 3.6-v7.2.0 + + # extras + # extra-deps includes reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 + - python-version: 3.8 + env: + TOXENV: extra-deps + - python-version: 3.9 + env: + TOXENV: asyncio + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system libraries + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxslt-dev + + - name: Run tests + env: ${{ matrix.env }} + run: | + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + pip install -U tox + tox + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml new file mode 100644 index 000000000..5459a845b --- /dev/null +++ b/.github/workflows/tests-windows.yml @@ -0,0 +1,36 @@ +name: Windows +on: [push, pull_request] + +jobs: + tests: + runs-on: windows-latest + strategy: + matrix: + include: + - python-version: 3.6 + env: + TOXENV: windows-pinned + - python-version: 3.7 + env: + TOXENV: py + - python-version: 3.8 + env: + TOXENV: py + # https://twistedmatrix.com/trac/ticket/9990 + #- python-version: 3.9 + #env: + #TOXENV: py + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.gitignore b/.gitignore index ff6e2ea65..d77d24624 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,12 @@ htmlcov/ .coverage .pytest_cache/ .coverage.* +coverage.* +test-output.* .cache/ +.mypy_cache/ +/tests/keys/localhost.crt +/tests/keys/localhost.key # Windows Thumbs.db diff --git a/.readthedocs.yml b/.readthedocs.yml index 17eba34f3..80a1cd036 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,11 +1,16 @@ version: 2 +formats: all sphinx: configuration: docs/conf.py fail_on_warning: true + +build: + image: latest + python: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image - version: 3.7 # Keep in sync with .travis.yml + version: 3.8 # Keep in sync with .github/workflows/checks.yml install: - requirements: docs/requirements.txt - path: . diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 66e1a9617..000000000 --- a/.travis.yml +++ /dev/null @@ -1,65 +0,0 @@ -language: python -dist: xenial -branches: - only: - - master - - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ -matrix: - include: - - env: TOXENV=security - python: 3.8 - - env: TOXENV=flake8 - python: 3.8 - - env: TOXENV=pypy3 - - env: TOXENV=py35 - python: 3.5 - - env: TOXENV=pinned - python: 3.5 - - env: TOXENV=py35-asyncio - python: 3.5.2 - - env: TOXENV=py36 - python: 3.6 - - env: TOXENV=py37 - python: 3.7 - - env: TOXENV=py38 - python: 3.8 - - env: TOXENV=extra-deps - python: 3.8 - - env: TOXENV=py38-asyncio - python: 3.8 - - env: TOXENV=docs - python: 3.7 # Keep in sync with .readthedocs.yml -install: - - | - if [ "$TOXENV" = "pypy3" ]; then - export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable" - wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - - pip install -U tox twine wheel codecov - -script: tox -after_success: - - codecov -notifications: - irc: - use_notice: true - skip_join: true - channels: - - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: scrapy - password: - secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA= - on: - tags: true - repo: scrapy/scrapy - condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" diff --git a/AUTHORS b/AUTHORS index bcaa1ecd3..9706adf42 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,8 +1,8 @@ Scrapy was brought to life by Shane Evans while hacking a scraping framework prototype for Mydeco (mydeco.com). It soon became maintained, extended and improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to -bootstrap the project. In mid-2011, Scrapinghub became the new official -maintainer. +bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new +official maintainer. Here is the list of the primary authors & contributors: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d1cd3e517..902cd523e 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -55,7 +55,7 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@zyte.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. @@ -72,3 +72,6 @@ available at [http://contributor-covenant.org/version/1/4][version]. [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/README.rst b/README.rst index ce5973bcd..5750e2c0f 100644 --- a/README.rst +++ b/README.rst @@ -10,9 +10,17 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg - :target: https://travis-ci.org/scrapy/scrapy - :alt: Build Status +.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu + :alt: Ubuntu + +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS + +.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows + :alt: Windows .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy @@ -34,13 +42,20 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other +contributors`_. + +.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Zyte: https://www.zyte.com/ + Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. + Requirements ============ -* Python 3.5+ +* Python 3.6+ * Works on Linux, Windows, macOS, BSD Install @@ -81,7 +96,7 @@ Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@scrapinghub.com. +Please report unacceptable behavior to opensource@zyte.com. Companies using Scrapy ====================== diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 7fd636864..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,25 +0,0 @@ -platform: x86 -version: '{branch}-{build}' -environment: - matrix: - - PYTHON: "C:\\Python36" - TOX_ENV: py36 - -branches: - only: - - master - - /d+\.\d+\.\d+[\w\-]*$/ - -install: - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%" - - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE" - - "pip install -U tox" - -build: false -skip_tags: true -test_script: - - "tox -e %TOX_ENV%" - -cache: - - '%LOCALAPPDATA%\pip\cache' diff --git a/conftest.py b/conftest.py index be5fbabf4..05b4ccdad 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,11 @@ from pathlib import Path import pytest +from twisted.web.http import H2_ENABLED + +from scrapy.utils.reactor import install_reactor + +from tests.keys import generate_keys def _py_files(folder): @@ -12,14 +17,23 @@ collect_ignore = [ "scrapy/utils/testsite.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), - # Py36-only parts of respective tests - *_py_files("tests/py36"), + # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess + *_py_files("tests/CrawlerRunner"), ] -for line in open('tests/ignores.txt'): - file_path = line.strip() - if file_path and file_path[0] != '#': - collect_ignore.append(file_path) +with open('tests/ignores.txt') as reader: + for line in reader: + file_path = line.strip() + if file_path and file_path[0] != '#': + collect_ignore.append(file_path) + +if not H2_ENABLED: + collect_ignore.extend( + ( + 'scrapy/core/downloader/handlers/http2.py', + *_py_files("scrapy/core/http2"), + ) + ) @pytest.fixture() @@ -38,6 +52,14 @@ def pytest_collection_modifyitems(session, config, items): pass +def pytest_addoption(parser): + parser.addoption( + "--reactor", + default="default", + choices=["default", "asyncio"], + ) + + @pytest.fixture(scope='class') def reactor_pytest(request): if not request.cls: @@ -51,3 +73,12 @@ def reactor_pytest(request): def only_asyncio(request, reactor_pytest): if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio': pytest.skip('This test is only run with --reactor=asyncio') + + +def pytest_configure(config): + if config.getoption("--reactor") == "asyncio": + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + +# Generate localhost certificate files, needed by some tests +generate_keys() diff --git a/docs/Makefile b/docs/Makefile index ff68bf1ae..87d5d3047 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,7 +8,7 @@ PYTHON = python SPHINXOPTS = PAPER = SOURCES = -SHELL = /bin/bash +SHELL = /usr/bin/env bash ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ -D latex_elements.papersize=$(PAPER) \ diff --git a/docs/README.rst b/docs/README.rst index 0a343cd19..0b7afa548 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -57,3 +57,12 @@ There is a way to recreate the doc automatically when you make changes, you need to install watchdog (``pip install watchdog``) and then use:: make watch + +Alternative method using tox +---------------------------- + +To compile the documentation to HTML run the following command:: + + tox -e docs + +Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir. diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 192123473..640660943 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -17,7 +17,7 @@ class SettingsListDirective(Directive): def is_setting_index(node): if node.tagname == 'index': # index entries for setting directives look like: - # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')] + # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] entry_type, info, refid = node['entries'][0][:3] return entry_type == 'pair' and info.endswith('; setting') return False diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 000000000..64f16939c --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,10 @@ +/* Move lists closer to their introducing paragraph */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} +.rst-content p + ol, .rst-content p + ul { + margin-top: -18px; /* Compensates margin-top: 24px of p */ +} +.rst-content dl p + ol, .rst-content dl p + ul { + margin-top: -6px; /* Compensates margin-top: 12px of p */ +} \ No newline at end of file diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index a6f6cbda8..18a5231ee 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -3,14 +3,9 @@ {% block footer %} {{ super() }} {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index 6e2399f66..406c4d94a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Scrapy documentation build configuration file, created by # sphinx-quickstart on Mon Nov 24 12:02:52 2008. # @@ -51,7 +49,7 @@ master_doc = 'index' # General information about the project. project = 'Scrapy' -copyright = '2008–{}, Scrapy developers'.format(datetime.now().year) +copyright = f'2008–{datetime.now().year}, Scrapy developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -102,6 +100,9 @@ exclude_trees = ['.build'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' +# List of Sphinx warnings that will not be raised +suppress_warnings = ['epub.unknown_project_files'] + # Options for HTML output # ----------------------- @@ -121,7 +122,6 @@ html_theme = 'sphinx_rtd_theme' import sphinx_rtd_theme html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # The style sheet to use for HTML and HTML Help pages. A file of that name # must exist either in Sphinx' static/ path, or in one of the custom paths # given in html_static_path. @@ -182,6 +182,10 @@ html_copy_source = True # Output file base name for HTML help builder. htmlhelp_basename = 'Scrapydoc' +html_css_files = [ + 'custom.css', +] + # Options for LaTeX output # ------------------------ @@ -280,8 +284,11 @@ coverage_ignore_pyobjects = [ # ------------------------------------- intersphinx_mapping = { + 'attrs': ('https://www.attrs.org/en/stable/', None), 'coverage': ('https://coverage.readthedocs.io/en/stable', None), + 'cryptography' : ('https://cryptography.io/en/latest/', None), 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), + 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), @@ -295,3 +302,23 @@ intersphinx_mapping = { # ------------------------------------ hoverxref_auto_ref = True +hoverxref_role_types = { + "class": "tooltip", + "confval": "tooltip", + "hoverxref": "tooltip", + "mod": "tooltip", + "ref": "tooltip", +} +hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal'] + + +def setup(app): + app.connect('autodoc-skip-member', maybe_skip_member) + + +def maybe_skip_member(app, what, name, obj, skip, options): + if not skip: + # autodocs was generating a text "alias of" for the following members + # https://github.com/sphinx-doc/sphinx/issues/4422 + return name in {'default_item_class', 'default_selector_class'} + return skip diff --git a/docs/contributing.rst b/docs/contributing.rst index aed5ab92e..4d2580a6c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -108,6 +108,11 @@ Well-written patches should: tox -e docs-coverage +* if you are removing deprecated code, first make sure that at least 1 year + (12 months) has passed since the release that introduced the deprecation. + See :ref:`deprecation-policy`. + + .. _submitting-patches: Submitting patches @@ -135,7 +140,7 @@ original pull request author hasn't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is -not considered rude as soon as the original author is acknowledged by keeping +not considered rude as long as the original author is acknowledged by keeping his/her commits. You can pull an existing pull request to a local branch @@ -155,6 +160,9 @@ Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports removal, etc) in separate commits from functional changes. This will make pull requests easier to review and more likely to get merged. + +.. _coding-style: + Coding style ============ @@ -163,7 +171,7 @@ Scrapy: * Unless otherwise specified, follow :pep:`8`. -* It's OK to use lines longer than 80 chars if it improves the code +* It's OK to use lines longer than 79 chars if it improves the code readability. * Don't put your name in the code you contribute; git provides enough @@ -191,6 +199,17 @@ In any case, if something is covered in a docstring, use the documentation instead of duplicating the docstring in files within the ``docs/`` directory. +Documentation updates that cover new or modified features must use Sphinx’s +:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use +``VERSION`` as version, we will replace it with the actual version right before +the corresponding release. When we release a new major or minor version of +Scrapy, we remove these directives if they are older than 3 years. + +Documentation about deprecated features must be removed as those features are +deprecated, so that new readers do not run into it. New deprecations and +deprecation removals are documented in the :ref:`release notes `. + + Tests ===== diff --git a/docs/faq.rst b/docs/faq.rst index 75a0f4864..9709885f6 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars .. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use -.. _faq-python-versions: - -What Python versions does Scrapy support? ------------------------------------------ - -Scrapy is supported under Python 3.5+ -under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). -Python 3 support was added in Scrapy 1.1. -PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5. -Python 2 support was dropped in Scrapy 2.0. - -.. note:: - For Python 3 support on Windows, it is recommended to use - Anaconda/Miniconda as :ref:`outlined in the installation guide `. Did Scrapy "steal" X from Django? --------------------------------- @@ -159,6 +145,41 @@ How can I make Scrapy consume less memory? See previous question. +How can I prevent memory errors due to many allowed domains? +------------------------------------------------------------ + +If you have a spider with a long list of +:attr:`~scrapy.spiders.Spider.allowed_domains` (e.g. 50,000+), consider +replacing the default +:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware +with a :ref:`custom spider middleware ` that requires +less memory. For example: + +- If your domain names are similar enough, use your own regular expression + instead joining the strings in + :attr:`~scrapy.spiders.Spider.allowed_domains` into a complex regular + expression. + +- If you can `meet the installation requirements`_, use pyre2_ instead of + Python’s re_ to compile your URL-filtering regular expression. See + :issue:`1908`. + +See also other suggestions at `StackOverflow`_. + +.. note:: Remember to disable + :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable + your custom implementation:: + + SPIDER_MIDDLEWARES = { + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, + 'myproject.middlewares.CustomOffsiteMiddleware': 500, + } + +.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation +.. _pyre2: https://github.com/andreasvc/pyre2 +.. _re: https://docs.python.org/library/re.html +.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 + Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- @@ -250,15 +271,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file? To dump into a JSON file:: - scrapy crawl myspider -o items.json + scrapy crawl myspider -O items.json To dump into a CSV file:: - scrapy crawl myspider -o items.csv + scrapy crawl myspider -O items.csv To dump into a XML file:: - scrapy crawl myspider -o items.xml + scrapy crawl myspider -O items.xml For more information see :ref:`topics-feed-exports` @@ -342,15 +363,15 @@ method for this purpose. For example:: from copy import deepcopy - from scrapy.item import BaseItem - + from itemadapter import is_item, ItemAdapter class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): for item in result: - if isinstance(item, (BaseItem, dict)): - for _ in range(item['multiply_by']): + if is_item(item): + adapter = ItemAdapter(item) + for _ in range(adapter['multiply_by']): yield deepcopy(item) Does Scrapy support IPv6 addresses? @@ -371,6 +392,20 @@ Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switch different reactor is possible by using the :setting:`TWISTED_REACTOR` setting. +.. _faq-stop-response-download: + +How can I cancel the download of a given response? +-------------------------------------------------- + +In some situations, it might be useful to stop the download of a certain response. +For instance, sometimes you can determine whether or not you need the full contents +of a response by inspecting its headers or the first bytes of its body. In that case, +you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received` +or :class:`~scrapy.signals.headers_received` signals and raising a +:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the +:ref:`topics-stop-response-download` topic for additional information and examples. + + .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) diff --git a/docs/index.rst b/docs/index.rst index 11aa5c9be..433798aa8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -78,7 +78,6 @@ Basic concepts topics/settings topics/exceptions - :doc:`topics/commands` Learn about the command-line tool used to manage your Scrapy project. @@ -228,6 +227,7 @@ Extending Scrapy topics/extensions topics/api topics/signals + topics/scheduler topics/exporters @@ -249,6 +249,9 @@ Extending Scrapy :doc:`topics/signals` See all available signals and how to work with them. +:doc:`topics/scheduler` + Understand the scheduler component. + :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 6356e0eea..8581dde0b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -4,12 +4,19 @@ Installation guide ================== +.. _faq-python-versions: + +Supported Python versions +========================= + +Scrapy requires Python 3.6+, either the CPython implementation (default) or +the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). + +.. _intro-install-scrapy: + Installing Scrapy ================= -Scrapy runs on Python 3.5 or above under CPython (default Python -implementation) and PyPy (starting with PyPy 5.9). - If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows and macOS. @@ -23,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, +to avoid conflicting with your system packages. + Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, -to avoid conflicting with your system packages. - For more detailed and platform specifics instructions, as well as troubleshooting information, read on. @@ -63,10 +70,9 @@ In case of any trouble related to these dependencies, please refer to their respective installation instructions: * `lxml installation`_ -* `cryptography installation`_ +* :doc:`cryptography installation ` .. _lxml installation: https://lxml.de/installation.html -.. _cryptography installation: https://cryptography.io/en/latest/installation/ .. _intro-using-virtualenv: @@ -112,6 +118,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: conda install -c conda-forge scrapy +To install Scrapy on Windows using ``pip``: + +.. warning:: + This installation method requires “Microsoft Visual C++” for installing some + Scrapy dependencies, which demands significantly more disk space than Anaconda. + +#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. + +#. Run the Visual Studio Installer. + +#. Under the Workloads section, select **C++ build tools**. + +#. Check the installation details and make sure following packages are selected as optional components: + + * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) + + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) + +#. Install the Visual Studio Build Tools. + +Now, you should be able to :ref:`install Scrapy ` using ``pip``. .. _intro-install-ubuntu: @@ -205,8 +232,8 @@ PyPy We recommend using the latest PyPy version. The version tested is 5.9.0. For PyPy3, only Linux installation was tested. -Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy. -This means that these dependecies will be built during installation. +Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. +This means that these dependencies will be built during installation. On macOS, you are likely to face an issue with building Cryptography dependency, solution to this problem is described `here `_, @@ -259,10 +286,10 @@ For details, see `Issue #2473 `_. .. _cryptography: https://cryptography.io/en/latest/ .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ .. _setuptools: https://pypi.python.org/pypi/setuptools -.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Scrapinghub: https://scrapinghub.com .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html +.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio +.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 01986b594..dd80c7bd0 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination:: if next_page is not None: yield response.follow(next_page, self.parse) - Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider quotes_spider.py -o quotes.json + scrapy runspider quotes_spider.py -o quotes.jl +When this finishes you will have in the ``quotes.jl`` file a list of the +quotes in JSON Lines format, containing text and author, looking like this:: -When this finishes you will have in the ``quotes.json`` file a list of the -quotes in JSON format, containing text and author, looking like this (reformatted -here for better readability):: - - [{ - "author": "Jane Austen", - "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" - }, - { - "author": "Groucho Marx", - "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" - }, - { - "author": "Steve Martin", - "text": "\u201cA day without sunshine is like, you know, night.\u201d" - }, - ...] + {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} + {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} + {"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"} + ... What just happened? diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 1768badbb..740e47d0c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -25,16 +25,16 @@ Scrapy. If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource. If you're new to programming and want to start with Python, the following books -may be useful to you: +may be useful to you: * `Automate the Boring Stuff With Python`_ -* `How To Think Like a Computer Scientist`_ +* `How To Think Like a Computer Scientist`_ -* `Learn Python 3 The Hard Way`_ +* `Learn Python 3 The Hard Way`_ You can also take a look at `this list of Python resources for non-programmers`_, -as well as the `suggested resources in the learnpython-subreddit`_. +as well as the `suggested resources in the learnpython-subreddit`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers @@ -62,7 +62,7 @@ This will create a ``tutorial`` directory with the following contents:: __init__.py items.py # project items definition file - + middlewares.py # project middlewares file pipelines.py # project pipelines file @@ -101,10 +101,10 @@ This is the code for our first Spider. Save it in a file named def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) - self.log('Saved file %s' % filename) + self.log(f'Saved file {filename}') As you can see, our Spider subclasses :class:`scrapy.Spider ` @@ -190,7 +190,7 @@ for your spider:: def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) @@ -287,8 +287,8 @@ to be scraped, you can at least get **some** data. Besides the :meth:`~scrapy.selector.SelectorList.getall` and :meth:`~scrapy.selector.SelectorList.get` methods, you can also use -the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular -expressions`_: +the :meth:`~scrapy.selector.SelectorList.re` method to extract using +:doc:`regular expressions `: >>> response.css('title::text').re(r'Quotes.*') ['Quotes to Scrape'] @@ -305,7 +305,6 @@ with a selector (see :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for visually selected elements, which works in many browsers. -.. _regular expressions: https://docs.python.org/3/library/re.html .. _Selector Gadget: https://selectorgadget.com/ @@ -406,8 +405,6 @@ to get all of them: from sys import version_info -.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output") - Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary: @@ -465,16 +462,15 @@ Storing the scraped data The simplest way to store the scraped data is by using :ref:`Feed exports `, with the following command:: - scrapy crawl quotes -o quotes.json + scrapy crawl quotes -O quotes.json -That will generate an ``quotes.json`` file containing all scraped items, +That will generate a ``quotes.json`` file containing all scraped items, serialized in `JSON`_. -For historic reasons, Scrapy appends to a given file instead of overwriting -its contents. If you run this command twice without removing the file -before the second time, you'll end up with a broken JSON file. - -You can also use other formats, like `JSON Lines`_:: +The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead +to append new content to any existing file. However, appending to a JSON file +makes the file contents invalid JSON. When appending to a file, consider +using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jl @@ -705,7 +701,7 @@ Using spider arguments You can provide command line arguments to your spiders by using the ``-a`` option when running them:: - scrapy crawl quotes -o quotes-humor.json -a tag=humor + scrapy crawl quotes -O quotes-humor.json -a tag=humor These arguments are passed to the Spider's ``__init__`` method and become spider attributes by default. diff --git a/docs/news.rst b/docs/news.rst index e9b7140cd..0ea412e75 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,994 @@ Release notes ============= +.. _release-2.5.0: + +Scrapy 2.5.0 (2021-04-06) +------------------------- + +Highlights: + +- Official Python 3.9 support + +- Experimental :ref:`HTTP/2 support ` + +- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function + to retry requests from spider callbacks + +- New :class:`~scrapy.signals.headers_received` signal that allows stopping + downloads early + +- New :class:`Response.protocol ` attribute + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and + had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`. + (:issue:`4901`) + +- Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment + variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`) + + +Deprecations +~~~~~~~~~~~~ + +- The :mod:`scrapy.utils.py36` module is now deprecated in favor of + :mod:`scrapy.utils.asyncgen`. (:issue:`4900`) + + +New features +~~~~~~~~~~~~ + +- Experimental :ref:`HTTP/2 support ` through a new download handler + that can be assigned to the ``https`` protocol in the + :setting:`DOWNLOAD_HANDLERS` setting. + (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) + +- The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request` + function may be used from spider callbacks or middlewares to handle the + retrying of a request beyond the scenarios that + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports. + (:issue:`3590`, :issue:`3685`, :issue:`4902`) + +- The new :class:`~scrapy.signals.headers_received` signal gives early access + to response headers and allows :ref:`stopping downloads + `. + (:issue:`1772`, :issue:`4897`) + +- The new :attr:`Response.protocol ` + attribute gives access to the string that identifies the protocol used to + download a response. (:issue:`4878`) + +- :ref:`Stats ` now include the following entries that indicate + the number of successes and failures in storing + :ref:`feeds `:: + + feedexport/success_count/ + feedexport/failed_count/ + + Where ```` is the feed storage backend class name, such as + :class:`~scrapy.extensions.feedexport.FileFeedStorage` or + :class:`~scrapy.extensions.feedexport.FTPFeedStorage`. + + (:issue:`3947`, :issue:`4850`) + +- The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider + middleware now logs ignored URLs with ``INFO`` :ref:`logging level + ` instead of ``DEBUG``, and it now includes the following entry + into :ref:`stats ` to keep track of the number of ignored + URLs:: + + urllength/request_ignored_count + + (:issue:`5036`) + +- The + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + downloader middleware now logs the number of decompressed responses and the + total count of resulting bytes:: + + httpcompression/response_bytes + httpcompression/response_count + + (:issue:`4797`, :issue:`4799`) + + +Bug fixes +~~~~~~~~~ + +- Fixed installation on PyPy installing PyDispatcher in addition to + PyPyDispatcher, which could prevent Scrapy from working depending on which + package got imported. (:issue:`4710`, :issue:`4814`) + +- When inspecting a callback to check if it is a generator that also returns + a value, an exception is no longer raised if the callback has a docstring + with lower indentation than the following code. + (:issue:`4477`, :issue:`4935`) + +- The `Content-Length `_ + header is no longer omitted from responses when using the default, HTTP/1.1 + download handler (see :setting:`DOWNLOAD_HANDLERS`). + (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`) + +- Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False`` + now has the same effect as not setting it at all, instead of having the + same effect as setting it to ``True``. + (:issue:`3851`, :issue:`4694`) + + +Documentation +~~~~~~~~~~~~~ + +- Added instructions to :ref:`install Scrapy in Windows using pip + `. + (:issue:`4715`, :issue:`4736`) + +- Logging documentation now includes :ref:`additional ways to filter logs + `. + (:issue:`4216`, :issue:`4257`, :issue:`4965`) + +- Covered how to deal with long lists of allowed domains in the :ref:`FAQ + `. (:issue:`2263`, :issue:`3667`) + +- Covered scrapy-bench_ in :ref:`benchmarking`. + (:issue:`4996`, :issue:`5016`) + +- Clarified that one :ref:`extension ` instance is created + per crawler. + (:issue:`5014`) + +- Fixed some errors in examples. + (:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`, + :issue:`5008`) + +- Fixed some external links, typos, and so on. + (:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`, + :issue:`5063`) + +- The :ref:`list of Request.meta keys ` is now sorted + alphabetically. + (:issue:`5061`, :issue:`5065`) + +- Updated references to Scrapinghub, which is now called Zyte. + (:issue:`4973`, :issue:`5072`) + +- Added a mention to contributors in the README. (:issue:`4956`) + +- Reduced the top margin of lists. (:issue:`4974`) + + +Quality Assurance +~~~~~~~~~~~~~~~~~ + +- Made Python 3.9 support official (:issue:`4757`, :issue:`4759`) + +- Extended typing hints (:issue:`4895`) + +- Fixed deprecated uses of the Twisted API. + (:issue:`4940`, :issue:`4950`, :issue:`5073`) + +- Made our tests run with the new pip resolver. + (:issue:`4710`, :issue:`4814`) + +- Added tests to ensure that :ref:`coroutine support ` + is tested. (:issue:`4987`) + +- Migrated from Travis CI to GitHub Actions. (:issue:`4924`) + +- Fixed CI issues. + (:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`, + :issue:`5053`) + +- Implemented code refactorings, style fixes and cleanups. + (:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`) + + +.. _release-2.4.1: + +Scrapy 2.4.1 (2020-11-17) +------------------------- + +- Fixed :ref:`feed exports ` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`) + +- Fixed the AsyncIO event loop handling, which could make code hang + (:issue:`4855`, :issue:`4872`) + +- Fixed the IPv6-capable DNS resolver + :class:`~scrapy.resolver.CachingHostnameResolver` for download handlers + that call + :meth:`reactor.resolve ` + (:issue:`4802`, :issue:`4803`) + +- Fixed the output of the :command:`genspider` command showing placeholders + instead of the import path of the generated spider module (:issue:`4874`) + +- Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`, + :issue:`4876`) + + +.. _release-2.4.0: + +Scrapy 2.4.0 (2020-10-11) +------------------------- + +Highlights: + +* Python 3.5 support has been dropped. + +* The ``file_path`` method of :ref:`media pipelines ` + can now access the source :ref:`item `. + + This allows you to set a download file path based on item data. + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` + +* You can now choose whether :ref:`feed exports ` + overwrite or append to the output file. + + For example, when using the :command:`crawl` or :command:`runspider` + commands, you can use the ``-O`` option instead of ``-o`` to overwrite the + output file. + +* Zstd-compressed responses are now supported if zstandard_ is installed. + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +* Python 3.6 or greater is now required; support for Python 3.5 has been + dropped + + As a result: + + - When using PyPy, PyPy 7.2.0 or greater :ref:`is now required + ` + + - For Amazon S3 storage support in :ref:`feed exports + ` or :ref:`media pipelines + `, botocore_ 1.4.87 or greater is now required + + - To use the :ref:`images pipeline `, Pillow_ 4.0.0 or + greater is now required + + (:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`, + :issue:`4764`) + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again + discards cookies defined in :attr:`Request.headers + `. + + We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it + was reported that the current implementation could break existing code. + + If you need to set cookies for a request, use the :class:`Request.cookies + ` parameter. + + A future version of Scrapy will include a new, better implementation of the + reverted bug fix. + + (:issue:`4717`, :issue:`4823`) + + +.. _2.4-deprecation-removals: + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the + values of ``access_key`` and ``secret_key`` from the running project + settings when they are not passed to its ``__init__`` method; you must + either pass those parameters to its ``__init__`` method or use + :class:`S3FeedStorage.from_crawler + ` + (:issue:`4356`, :issue:`4411`, :issue:`4688`) + +* :attr:`Rule.process_request ` + no longer admits callables which expect a single ``request`` parameter, + rather than both ``request`` and ``response`` (:issue:`4818`) + + +Deprecations +~~~~~~~~~~~~ + +* In custom :ref:`media pipelines `, signatures that + do not accept a keyword-only ``item`` parameter in any of the methods that + :ref:`now support this parameter ` are now + deprecated (:issue:`4628`, :issue:`4686`) + +* In custom :ref:`feed storage backend classes `, + ``__init__`` method signatures that do not accept a keyword-only + ``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`, + :issue:`4512`) + +* The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated + (:issue:`4684`, :issue:`4701`) + +* The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use + :func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`, + :issue:`4776`) + + +New features +~~~~~~~~~~~~ + +.. _media-pipeline-item-parameter: + +* The following methods of :ref:`media pipelines ` now + accept an ``item`` keyword-only parameter containing the source + :ref:`item `: + + - In :class:`scrapy.pipelines.files.FilesPipeline`: + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_path` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download` + + - In :class:`scrapy.pipelines.images.ImagesPipeline`: + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download` + + (:issue:`4628`, :issue:`4686`) + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` (:issue:`4606`, :issue:`4768`) + +* :ref:`Feed exports ` gained overwrite support: + + * When using the :command:`crawl` or :command:`runspider` commands, you + can use the ``-O`` option instead of ``-o`` to overwrite the output + file + + * You can use the ``overwrite`` key in the :setting:`FEEDS` setting to + configure whether to overwrite the output file (``True``) or append to + its content (``False``) + + * The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage + backend classes ` now receive a new keyword-only + parameter, ``feed_options``, which is a dictionary of :ref:`feed + options ` + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* Zstd-compressed responses are now supported if zstandard_ is installed + (:issue:`4831`) + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead (:issue:`3870`, :issue:`3873`). + + This includes also settings where only part of its value is made of an + import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or + :setting:`DOWNLOAD_HANDLERS`. + +* :ref:`Downloader middlewares ` can now + override :class:`response.request `. + + If a :ref:`downloader middleware ` returns + a :class:`~scrapy.http.Response` object from + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` + or + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` + with a custom :class:`~scrapy.http.Request` object assigned to + :class:`response.request `: + + - The response is handled by the callback of that custom + :class:`~scrapy.http.Request` object, instead of being handled by the + callback of the original :class:`~scrapy.http.Request` object + + - That custom :class:`~scrapy.http.Request` object is now sent as the + ``request`` argument to the :signal:`response_received` signal, instead + of the original :class:`~scrapy.http.Request` object + + (:issue:`4529`, :issue:`4632`) + +* When using the :ref:`FTP feed storage backend `: + + - It is now possible to set the new ``overwrite`` :ref:`feed option + ` to ``False`` to append to an existing file instead of + overwriting it + + - The FTP password can now be omitted if it is not necessary + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now + supports an ``errors`` parameter to indicate how to handle encoding errors + (:issue:`4755`) + +* When :ref:`using asyncio `, it is now possible to + :ref:`set a custom asyncio loop ` (:issue:`4306`, + :issue:`4414`) + +* Serialized requests (see :ref:`topics-jobs`) now support callbacks that are + spider methods that delegate on other callable (:issue:`4756`) + +* When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged + message is now a warning, instead of an error (:issue:`3874`, + :issue:`3886`, :issue:`4752`) + + +Bug fixes +~~~~~~~~~ + +* The :command:`genspider` command no longer overwrites existing files + unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`, + :issue:`4623`) + +* Cookies with an empty value are no longer considered invalid cookies + (:issue:`4772`) + +* The :command:`runspider` command now supports files with the ``.pyw`` file + extension (:issue:`4643`, :issue:`4646`) + +* The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + middleware now simply ignores unsupported proxy values (:issue:`3331`, + :issue:`4778`) + +* Checks for generator callbacks with a ``return`` statement no longer warn + about ``return`` statements in nested functions (:issue:`4720`, + :issue:`4721`) + +* The system file mode creation mask no longer affects the permissions of + files generated using the :command:`startproject` command (:issue:`4722`) + +* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names + (:issue:`861`, :issue:`4746`) + +* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can + work when using a headless browser (:issue:`4835`) + + +Documentation +~~~~~~~~~~~~~ + +* The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`, + :issue:`4724`) + +* Improved the documentation of + :ref:`link extractors ` with an usage example from + a spider callback and reference documentation for the + :class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`) + +* Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the + :class:`~scrapy.extensions.closespider.CloseSpider` extension + (:issue:`4836`) + +* Removed references to Python 2’s ``unicode`` type (:issue:`4547`, + :issue:`4703`) + +* We now have an :ref:`official deprecation policy ` + (:issue:`4705`) + +* Our :ref:`documentation policies ` now cover usage + of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged` + directives, and we have removed usages referencing Scrapy 1.4.0 and earlier + versions (:issue:`3971`, :issue:`4310`) + +* Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`, + :issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Extended typing hints (:issue:`4243`, :issue:`4691`) + +* Added tests for the :command:`check` command (:issue:`4663`) + +* Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`) + +* Improved Windows test coverage (:issue:`4723`) + +* Switched to :ref:`formatted string literals ` where possible + (:issue:`4307`, :issue:`4324`, :issue:`4672`) + +* Modernized :func:`super` usage (:issue:`4707`) + +* Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`, + :issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`, + :issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`, + :issue:`4820`, :issue:`4822`, :issue:`4839`) + + +.. _release-2.3.0: + +Scrapy 2.3.0 (2020-08-04) +------------------------- + +Highlights: + +* :ref:`Feed exports ` now support :ref:`Google Cloud + Storage ` as a storage backend + +* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver + output items in batches of up to the specified number of items. + + It also serves as a workaround for :ref:`delayed file delivery + `, which causes Scrapy to only start item delivery + after the crawl has finished when using certain storage backends + (:ref:`S3 `, :ref:`FTP `, + and now :ref:`GCS `). + +* The base implementation of :ref:`item loaders ` has been + moved into a separate library, :doc:`itemloaders `, + allowing usage from outside Scrapy and a separate release schedule + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* Removed the following classes and their parent modules from + ``scrapy.linkextractors``: + + * ``htmlparser.HtmlParserLinkExtractor`` + * ``regex.RegexLinkExtractor`` + * ``sgml.BaseSgmlLinkExtractor`` + * ``sgml.SgmlLinkExtractor`` + + Use + :class:`LinkExtractor ` + instead (:issue:`4356`, :issue:`4679`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated + (:issue:`4683`) + + +New features +~~~~~~~~~~~~ + +* :ref:`Feed exports ` support :ref:`Google Cloud + Storage ` (:issue:`685`, :issue:`3608`) + +* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries + (:issue:`4250`, :issue:`4434`) + +* The :command:`parse` command now allows specifying an output file + (:issue:`4317`, :issue:`4377`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support + ``--data-raw`` (:issue:`4612`) + +* A ``parse`` callback may now be used in built-in spider subclasses, such + as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`, + :issue:`781`, :issue:`4254` ) + + +Bug fixes +~~~~~~~~~ + +* Fixed the :ref:`CSV exporting ` of + :ref:`dataclass items ` and :ref:`attr.s items + ` (:issue:`4667`, :issue:`4668`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request + method to ``POST`` when a request body is specified and no request method + is specified (:issue:`4612`) + +* The processing of ANSI escape sequences in enabled in Windows 10.0.14393 + and later, where it is required for colored output (:issue:`4393`, + :issue:`4403`) + + +Documentation +~~~~~~~~~~~~~ + +* Updated the `OpenSSL cipher list format`_ link in the documentation about + the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`) + +* Simplified the code example in :ref:`topics-loaders-dataclass` + (:issue:`4652`) + +.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* The base implementation of :ref:`item loaders ` has been + moved into :doc:`itemloaders ` (:issue:`4005`, + :issue:`4516`) + +* Fixed a silenced error in some scheduler tests (:issue:`4644`, + :issue:`4645`) + +* Renewed the localhost certificate used for SSL tests (:issue:`4650`) + +* Removed cookie-handling code specific to Python 2 (:issue:`4682`) + +* Stopped using Python 2 unicode literal syntax (:issue:`4704`) + +* Stopped using a backlash for line continuation (:issue:`4673`) + +* Removed unneeded entries from the MyPy exception list (:issue:`4690`) + +* Automated tests now pass on Windows as part of our continuous integration + system (:issue:`4458`) + +* Automated tests now pass on the latest PyPy version for supported Python + versions in our continuous integration system (:issue:`4504`) + + +.. _release-2.2.1: + +Scrapy 2.2.1 (2020-07-17) +------------------------- + +* The :command:`startproject` command no longer makes unintended changes to + the permissions of files in the destination folder, such as removing + execution permissions (:issue:`4662`, :issue:`4666`) + + +.. _release-2.2.0: + +Scrapy 2.2.0 (2020-06-24) +------------------------- + +Highlights: + +* Python 3.5.2+ is required now +* :ref:`dataclass objects ` and + :ref:`attrs objects ` are now valid :ref:`item types + ` +* New :meth:`TextResponse.json ` method +* New :signal:`bytes_received` signal that allows canceling response download +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` fixes + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Support for Python 3.5.0 and 3.5.1 has been dropped; Scrapy now refuses to + run with a Python version lower than 3.5.2, which introduced + :class:`typing.Type` (:issue:`4615`) + + +Deprecations +~~~~~~~~~~~~ + +* :meth:`TextResponse.body_as_unicode + ` is now deprecated, use + :attr:`TextResponse.text ` instead + (:issue:`4546`, :issue:`4555`, :issue:`4579`) + +* :class:`scrapy.item.BaseItem` is now deprecated, use + :class:`scrapy.item.Item` instead (:issue:`4534`) + + +New features +~~~~~~~~~~~~ + +* :ref:`dataclass objects ` and + :ref:`attrs objects ` are now valid :ref:`item types + `, and a new itemadapter_ library makes it easy to + write code that :ref:`supports any item type ` + (:issue:`2749`, :issue:`2807`, :issue:`3761`, :issue:`3881`, :issue:`4642`) + +* A new :meth:`TextResponse.json ` method + allows to deserialize JSON responses (:issue:`2444`, :issue:`4460`, + :issue:`4574`) + +* A new :signal:`bytes_received` signal allows monitoring response download + progress and :ref:`stopping downloads ` + (:issue:`4205`, :issue:`4559`) + +* The dictionaries in the result list of a :ref:`media pipeline + ` now include a new key, ``status``, which indicates + if the file was downloaded or, if the file was not downloaded, why it was + not downloaded; see :meth:`FilesPipeline.get_media_requests + ` for more + information (:issue:`2893`, :issue:`4486`) + +* When using :ref:`Google Cloud Storage ` for + a :ref:`media pipeline `, a warning is now logged if + the configured credentials do not grant the required permissions + (:issue:`4346`, :issue:`4508`) + +* :ref:`Link extractors ` are now serializable, + as long as you do not use :ref:`lambdas ` for parameters; for + example, you can now pass link extractors in :attr:`Request.cb_kwargs + ` or + :attr:`Request.meta ` when :ref:`persisting + scheduled requests ` (:issue:`4554`) + +* Upgraded the :ref:`pickle protocol ` that Scrapy uses + from protocol 2 to protocol 4, improving serialization capabilities and + performance (:issue:`4135`, :issue:`4541`) + +* :func:`scrapy.utils.misc.create_instance` now raises a :exc:`TypeError` + exception if the resulting instance is ``None`` (:issue:`4528`, + :issue:`4532`) + +.. _itemadapter: https://github.com/scrapy/itemadapter + + +Bug fixes +~~~~~~~~~ + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer + discards cookies defined in :attr:`Request.headers + ` (:issue:`1992`, :issue:`2400`) + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer + re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter + of the ``__init__`` method of :class:`~scrapy.http.Request` + (:issue:`2400`, :issue:`3575`) + +* When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is + ``False`` and the crawl yields no items, Scrapy no longer stops feed + exports after the first URI (:issue:`4621`, :issue:`4626`) + +* :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine + syntax ` no longer need to return an iterable, and may + instead return a :class:`~scrapy.http.Request` object, an + :ref:`item `, or ``None`` (:issue:`4609`) + +* The :command:`startproject` command now ensures that the generated project + folders and files have the right permissions (:issue:`4604`) + +* Fix a :exc:`KeyError` exception being sometimes raised from + :class:`scrapy.utils.datatypes.LocalWeakReferencedCache` (:issue:`4597`, + :issue:`4599`) + +* When :setting:`FEEDS` defines multiple URIs, log messages about items being + stored now contain information from the corresponding feed, instead of + always containing information about only one of the feeds (:issue:`4619`, + :issue:`4629`) + + +Documentation +~~~~~~~~~~~~~ + +* Added a new section about :ref:`accessing cb_kwargs from errbacks + ` (:issue:`4598`, :issue:`4634`) + +* Covered chompjs_ in :ref:`topics-parsing-javascript` (:issue:`4556`, + :issue:`4562`) + +* Removed from :doc:`topics/coroutines` the warning about the API being + experimental (:issue:`4511`, :issue:`4513`) + +* Removed references to unsupported versions of :doc:`Twisted + ` (:issue:`4533`) + +* Updated the description of the :ref:`screenshot pipeline example + `, which now uses :doc:`coroutine syntax + ` instead of returning a + :class:`~twisted.internet.defer.Deferred` (:issue:`4514`, :issue:`4593`) + +* Removed a misleading import line from the + :func:`scrapy.utils.log.configure_logging` code example (:issue:`4510`, + :issue:`4587`) + +* The display-on-hover behavior of internal documentation references now also + covers links to :ref:`commands `, :attr:`Request.meta + ` keys, :ref:`settings ` and + :ref:`signals ` (:issue:`4495`, :issue:`4563`) + +* It is again possible to download the documentation for offline reading + (:issue:`4578`, :issue:`4585`) + +* Removed backslashes preceding ``*args`` and ``**kwargs`` in some function + and method signatures (:issue:`4592`, :issue:`4596`) + +.. _chompjs: https://github.com/Nykakin/chompjs + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Adjusted the code base further to our :ref:`style guidelines + ` (:issue:`4237`, :issue:`4525`, :issue:`4538`, + :issue:`4539`, :issue:`4540`, :issue:`4542`, :issue:`4543`, :issue:`4544`, + :issue:`4545`, :issue:`4557`, :issue:`4558`, :issue:`4566`, :issue:`4568`, + :issue:`4572`) + +* Removed remnants of Python 2 support (:issue:`4550`, :issue:`4553`, + :issue:`4568`) + +* Improved code sharing between the :command:`crawl` and :command:`runspider` + commands (:issue:`4548`, :issue:`4552`) + +* Replaced ``chain(*iterable)`` with ``chain.from_iterable(iterable)`` + (:issue:`4635`) + +* You may now run the :mod:`asyncio` tests with Tox on any Python version + (:issue:`4521`) + +* Updated test requirements to reflect an incompatibility with pytest 5.4 and + 5.4.1 (:issue:`4588`) + +* Improved :class:`~scrapy.spiderloader.SpiderLoader` test coverage for + scenarios involving duplicate spider names (:issue:`4549`, :issue:`4560`) + +* Configured Travis CI to also run the tests with Python 3.5.2 + (:issue:`4518`, :issue:`4615`) + +* Added a `Pylint `_ job to Travis CI + (:issue:`3727`) + +* Added a `Mypy `_ job to Travis CI (:issue:`4637`) + +* Made use of set literals in tests (:issue:`4573`) + +* Cleaned up the Travis CI configuration (:issue:`4517`, :issue:`4519`, + :issue:`4522`, :issue:`4537`) + + +.. _release-2.1.0: + +Scrapy 2.1.0 (2020-04-24) +------------------------- + +Highlights: + +* New :setting:`FEEDS` setting to export to multiple feeds +* New :attr:`Response.ip_address ` attribute + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :exc:`AssertionError` exceptions triggered by :ref:`assert ` + statements have been replaced by new exception types, to support running + Python in optimized mode (see :option:`-O`) without changing Scrapy’s + behavior in any unexpected ways. + + If you catch an :exc:`AssertionError` exception from Scrapy, update your + code to catch the corresponding new exception. + + (:issue:`4440`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* The ``LOG_UNSERIALIZABLE_REQUESTS`` setting is no longer supported, use + :setting:`SCHEDULER_DEBUG` instead (:issue:`4385`) + +* The ``REDIRECT_MAX_METAREFRESH_DELAY`` setting is no longer supported, use + :setting:`METAREFRESH_MAXDELAY` instead (:issue:`4385`) + +* The :class:`~scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` + middleware has been removed, including the entire + :class:`scrapy.downloadermiddlewares.chunked` module; chunked transfers + work out of the box (:issue:`4431`) + +* The ``spiders`` property has been removed from + :class:`~scrapy.crawler.Crawler`, use :class:`CrawlerRunner.spider_loader + ` or instantiate + :setting:`SPIDER_LOADER_CLASS` with your settings instead (:issue:`4398`) + +* The ``MultiValueDict``, ``MultiValueDictKeyError``, and ``SiteNode`` + classes have been removed from :mod:`scrapy.utils.datatypes` + (:issue:`4400`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``FEED_FORMAT`` and ``FEED_URI`` settings have been deprecated in + favor of the new :setting:`FEEDS` setting (:issue:`1336`, :issue:`3858`, + :issue:`4507`) + + +New features +~~~~~~~~~~~~ + +* A new setting, :setting:`FEEDS`, allows configuring multiple output feeds + with different settings each (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The :command:`crawl` and :command:`runspider` commands now support multiple + ``-o`` parameters (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The :command:`crawl` and :command:`runspider` commands now support + specifying an output format by appending ``:`` to the output file + (:issue:`1336`, :issue:`3858`, :issue:`4507`) + +* The new :attr:`Response.ip_address ` + attribute gives access to the IP address that originated a response + (:issue:`3903`, :issue:`3940`) + +* A warning is now issued when a value in + :attr:`~scrapy.spiders.Spider.allowed_domains` includes a port + (:issue:`50`, :issue:`3198`, :issue:`4413`) + +* Zsh completion now excludes used option aliases from the completion list + (:issue:`4438`) + + +Bug fixes +~~~~~~~~~ + +* :ref:`Request serialization ` no longer breaks for + callbacks that are spider attributes which are assigned a function with a + different name (:issue:`4500`) + +* ``None`` values in :attr:`~scrapy.spiders.Spider.allowed_domains` no longer + cause a :exc:`TypeError` exception (:issue:`4410`) + +* Zsh completion no longer allows options after arguments (:issue:`4438`) + +* zope.interface 5.0.0 and later versions are now supported + (:issue:`4447`, :issue:`4448`) + +* :meth:`Spider.make_requests_from_url + `, deprecated in Scrapy + 1.4.0, now issues a warning when used (:issue:`4412`) + + +Documentation +~~~~~~~~~~~~~ + +* Improved the documentation about signals that allow their handlers to + return a :class:`~twisted.internet.defer.Deferred` (:issue:`4295`, + :issue:`4390`) + +* Our PyPI entry now includes links for our documentation, our source code + repository and our issue tracker (:issue:`4456`) + +* Covered the `curl2scrapy `_ + service in the documentation (:issue:`4206`, :issue:`4455`) + +* Removed references to the Guppy library, which only works in Python 2 + (:issue:`4285`, :issue:`4343`) + +* Extended use of InterSphinx to link to Python 3 documentation + (:issue:`4444`, :issue:`4445`) + +* Added support for Sphinx 3.0 and later (:issue:`4475`, :issue:`4480`, + :issue:`4496`, :issue:`4503`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Removed warnings about using old, removed settings (:issue:`4404`) + +* Removed a warning about importing + :class:`~twisted.internet.testing.StringTransport` from + ``twisted.test.proto_helpers`` in Twisted 19.7.0 or newer (:issue:`4409`) + +* Removed outdated Debian package build files (:issue:`4384`) + +* Removed :class:`object` usage as a base class (:issue:`4430`) + +* Removed code that added support for old versions of Twisted that we no + longer support (:issue:`4472`) + +* Fixed code style issues (:issue:`4468`, :issue:`4469`, :issue:`4471`, + :issue:`4481`) + +* Removed :func:`twisted.internet.defer.returnValue` calls (:issue:`4443`, + :issue:`4446`, :issue:`4489`) + + .. _release-2.0.1: Scrapy 2.0.1 (2020-03-18) @@ -631,6 +1619,8 @@ Deprecation removals * ``scrapy.xlib`` has been removed (:issue:`4015`) +.. _1.8-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -987,6 +1977,8 @@ The following deprecated settings have also been removed (:issue:`3578`): * ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`) +.. _1.7-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -1626,7 +2618,7 @@ Bug fixes - Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`). - Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`). - Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``, - ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). + ``DOWNLOADER_MIDDLEWARES``, ``EXTENSIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). Documentation ~~~~~~~~~~~~~ @@ -3382,7 +4374,7 @@ API changes - ``url`` and ``body`` attributes of Request objects are now read-only (#230) - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) -- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) +- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) - Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself. - Changes to Scrapy Manager (now called "Crawler"): - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` @@ -3529,13 +4521,14 @@ First release of Scrapy. .. _resource: https://docs.python.org/2/library/resource.html .. _robots.txt: https://www.robotstxt.org/ .. _scrapely: https://github.com/scrapy/scrapely +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ .. _Twisted: https://twistedmatrix.com/trac/ -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _w3lib: https://github.com/scrapy/w3lib .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 .. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/ .. _Zsh: https://www.zsh.org/ +.. _zstandard: https://pypi.org/project/zstandard/ diff --git a/docs/requirements.txt b/docs/requirements.txt index 773b92cea..a0930ba1e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -Sphinx>=2.1 -sphinx-hoverxref -sphinx-notfound-page -sphinx_rtd_theme +Sphinx>=3.0 +sphinx-hoverxref>=0.2b1 +sphinx-notfound-page>=0.4 +sphinx-rtd-theme>=0.5.2 \ No newline at end of file diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 1c461a511..445b2979f 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -4,8 +4,6 @@ Core API ======== -.. versionadded:: 0.15 - This section documents the Scrapy core API, and it's intended for developers of extensions and middlewares. @@ -91,7 +89,7 @@ how you :ref:`configure the downloader middlewares provided while constructing the crawler, and it is created after the arguments given in the :meth:`crawl` method. - .. method:: crawl(\*args, \**kwargs) + .. method:: crawl(*args, **kwargs) Starts the crawler by instantiating its spider class with the given ``args`` and ``kwargs`` arguments, while setting the execution engine in diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index ae25dfa2f..71d027c86 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The Scheduler receives requests from the engine and enqueues them for feeding -them later (also to the engine) when the engine requests them. +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine +requests them. .. _component-downloader: @@ -104,7 +105,7 @@ Spiders ------- Spiders are custom classes written by Scrapy users to parse responses and -extract items (aka scraped items) from them or additional requests to +extract :ref:`items ` from them or additional requests to follow. For more information see :ref:`topics-spiders`. .. _component-pipelines: diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 038a459fd..82c5f271f 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -1,16 +1,19 @@ +.. _using-asyncio: + ======= asyncio ======= .. versionadded:: 2.0 -Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio -reactor `, you may use :mod:`asyncio` and +Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the +asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy - versions may introduce related changes without a deprecation - period or warning. +.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet + recommended for production environments. Future Scrapy versions + may introduce related changes without a deprecation period or + warning. .. _install-asyncio: @@ -26,3 +29,15 @@ reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`:: install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to +use it instead of the default asyncio event loop. + + + diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 4317019fc..8e6aae65c 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies. AUTOTHROTTLE_TARGET_CONCURRENCY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 1.1 - Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 99469ebf1..0643df6a6 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -4,8 +4,6 @@ Benchmarking ============ -.. versionadded:: 0.17 - Scrapy comes with a simple benchmarking suite that spawns a local HTTP server and crawls it at the maximum possible speed. The goal of this benchmarking is to get an idea of how Scrapy performs in your hardware, in order to have a @@ -83,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -In the future, more cases will be added to the benchmarking suite to cover -other common scenarios. +Use scrapy-bench_ for more complex benchmarking. + +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench \ No newline at end of file diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index a0dcba90d..eef6b36ff 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,8 +6,6 @@ Command line tool ================= -.. versionadded:: 0.10 - Scrapy is controlled through the ``scrapy`` command-line tool, to be referred here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -468,7 +466,7 @@ Supported options: * ``--callback`` or ``-c``: spider method to use as callback for parsing the response -* ``--meta`` or ``-m``: additional request meta that will be passed to the callback +* ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' * ``--cbkwargs``: additional keyword arguments that will be passed to the callback. @@ -491,6 +489,10 @@ Supported options: * ``--verbose`` or ``-v``: display information for each depth level +* ``--output`` or ``-o``: dump scraped items to a file + + .. versionadded:: 2.3 + .. skip: start Usage example:: @@ -562,8 +564,6 @@ and Platform info, which is useful for bug reports. bench ----- -.. versionadded:: 0.17 - * Syntax: ``scrapy bench`` * Requires project: *no* @@ -598,8 +598,6 @@ Example: Register commands via setup.py entry points ------------------------------------------- -.. note:: This is an experimental feature, use with caution. - You can also add Scrapy commands from an external library by adding a ``scrapy.commands`` section in the entry points of the library ``setup.py`` file. diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 43db8f101..e61421bf1 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -4,8 +4,6 @@ Spiders Contracts ================= -.. versionadded:: 0.15 - Testing spiders can get particularly annoying and while nothing prevents you from writing unit tests the task gets cumbersome quickly. Scrapy offers an integrated way of testing your spiders by the means of contracts. @@ -78,10 +76,10 @@ override three methods: .. module:: scrapy.contracts -.. class:: Contract(method, \*args) +.. class:: Contract(method, *args) :param method: callback function to which the contract is associated - :type method: function + :type method: collections.abc.Callable :param args: list of arguments passed into the docstring (whitespace separated) @@ -136,7 +134,7 @@ Detecting check runs ==================== When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is -set to the ``true`` string. You can use `os.environ`_ to perform any change to +set to the ``true`` string. You can use :data:`os.environ` to perform any change to your spiders or your settings when ``scrapy check`` is used:: import os @@ -148,5 +146,3 @@ your spiders or your settings when ``scrapy check`` is used:: def __init__(self): if os.environ.get('SCRAPY_CHECK'): pass # Do some scraper adjustments when a check is running - -.. _os.environ: https://docs.python.org/3/library/os.html#os.environ diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 487cf4c6c..3b1549bd3 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -7,10 +7,6 @@ Coroutines Scrapy has :ref:`partial support ` for the :ref:`coroutine syntax `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy - versions may introduce related API and behavior changes without a - deprecation period or warning. - .. _coroutine-support: Supported callables @@ -21,19 +17,14 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.http.Request` callbacks. - The following are known caveats of the current implementation that we aim - to address in future versions of Scrapy: - - - The callback output is not processed until the whole callback finishes. + .. note:: The callback output is not processed until the whole callback + finishes. As a side effect, if the callback raises an exception, none of its output is processed. - - Because `asynchronous generators were introduced in Python 3.6`_, you - can only use ``yield`` if you are using Python 3.6 or later. - - If you need to output multiple items or requests and you are using - Python 3.5, return an iterable (e.g. a list) instead. + This is a known caveat of the current implementation that we aim to + address in a future version of Scrapy. - The :meth:`process_item` method of :ref:`item pipelines `. @@ -48,8 +39,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. -.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/ - Usage ===== @@ -57,27 +46,34 @@ There are several use cases for coroutines in Scrapy. Code that would return Deferreds when written for previous Scrapy versions, such as downloader middlewares and signal handlers, can be rewritten to be shorter and cleaner:: + from itemadapter import ItemAdapter + class DbPipeline: def _update_item(self, data, item): - item['field'] = data + adapter = ItemAdapter(item) + adapter['field'] = data return item def process_item(self, item, spider): - dfd = db.get_some_data(item['id']) + adapter = ItemAdapter(item) + dfd = db.get_some_data(adapter['id']) dfd.addCallback(self._update_item, item) return dfd becomes:: + from itemadapter import ItemAdapter + class DbPipeline: async def process_item(self, item, spider): - item['field'] = await db.get_some_data(item['id']) + adapter = ItemAdapter(item) + adapter['field'] = await db.get_some_data(adapter['id']) return item Coroutines may be used to call asynchronous code. This includes other coroutines, functions that return Deferreds and functions that return -`awaitable objects`_ such as :class:`~asyncio.Future`. This means you can use -many useful Python libraries providing such code:: +:term:`awaitable objects ` such as :class:`~asyncio.Future`. +This means you can use many useful Python libraries providing such code:: class MySpider(Spider): # ... @@ -107,4 +103,3 @@ Common use cases for asynchronous code include: :ref:`the screenshot pipeline example`). .. _aio-libs: https://github.com/aio-libs -.. _awaitable objects: https://docs.python.org/3/glossary.html#term-awaitable diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index 361914a29..961d6dc01 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -14,7 +14,7 @@ spiders come in. Popular choices for deploying Scrapy spiders are: * :ref:`Scrapyd ` (open source) -* :ref:`Scrapy Cloud ` (cloud-based) +* :ref:`Zyte Scrapy Cloud ` (cloud-based) .. _deploy-scrapyd: @@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers. .. _deploy-scrapy-cloud: -Deploying to Scrapy Cloud -========================= +Deploying to Zyte Scrapy Cloud +============================== -`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_, -the company behind Scrapy. +`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company +behind Scrapy. -Scrapy Cloud removes the need to setup and monitor servers -and provides a nice UI to manage spiders and review scraped items, -logs and stats. +Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a +nice UI to manage spiders and review scraped items, logs and stats. -To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool. -Please refer to the `Scrapy Cloud documentation`_ for more information. +To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line +tool. +Please refer to the `Zyte Scrapy Cloud documentation`_ for more information. -Scrapy Cloud is compatible with Scrapyd and one can switch between +Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between them as needed - the configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``. -.. _Scrapyd: https://github.com/scrapy/scrapyd .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud +.. _Scrapyd: https://github.com/scrapy/scrapyd .. _scrapyd-client: https://github.com/scrapy/scrapyd-client -.. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html -.. _Scrapinghub: https://scrapinghub.com/ +.. _shub: https://shub.readthedocs.io/en/latest/ +.. _Zyte: https://zyte.com/ +.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/ +.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index f1b0964c6..c83b1a9d9 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping ================================================= Here is a general guide on how to use your browser's Developer Tools -to ease the scraping process. Today almost all browsers come with +to ease the scraping process. Today almost all browsers come with built in `Developer Tools`_ and although we will use Firefox in this -guide, the concepts are applicable to any other browser. +guide, the concepts are applicable to any other browser. In this guide we'll introduce the basic tools to use from a browser's Developer Tools by scraping `quotes.toscrape.com`_. @@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things: Inspecting a website ==================== -By far the most handy feature of the Developer Tools is the `Inspector` -feature, which allows you to inspect the underlying HTML code of -any webpage. To demonstrate the Inspector, let's look at the +By far the most handy feature of the Developer Tools is the `Inspector` +feature, which allows you to inspect the underlying HTML code of +any webpage. To demonstrate the Inspector, let's look at the `quotes.toscrape.com`_-site. On the site we have a total of ten quotes from various authors with specific -tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes -on this page, without any meta-information about authors, tags, etc. +tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes +on this page, without any meta-information about authors, tags, etc. -Instead of viewing the whole source code for the page, we can simply right click +Instead of viewing the whole source code for the page, we can simply right click on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`. In it you should see something like this: @@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] -Adding ``text()`` at the end we are able to extract the first quote with this +Adding ``text()`` at the end we are able to extract the first quote with this basic selector. But this XPath is not really that clever. All it does is -go down a desired path in the source code starting from ``html``. So let's -see if we can refine our XPath a bit: +go down a desired path in the source code starting from ``html``. So let's +see if we can refine our XPath a bit: -If we check the `Inspector` again we'll see that directly beneath our -expanded ``div`` tag we have nine identical ``div`` tags, each with the -same attributes as our first. If we expand any of them, we'll see the same +If we check the `Inspector` again we'll see that directly beneath our +expanded ``div`` tag we have nine identical ``div`` tags, each with the +same attributes as our first. If we expand any of them, we'll see the same structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can -expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and +expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and see each quote: .. code-block:: html @@ -121,7 +121,7 @@ see each quote: With this knowledge we can refine our XPath: Instead of a path to follow, -we'll simply select all ``span`` tags with the ``class="text"`` by using +we'll simply select all ``span`` tags with the ``class="text"`` by using the `has-class-extension`_: >>> response.xpath('//span[has-class("text")]/text()').getall() @@ -130,45 +130,45 @@ the `has-class-extension`_: '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', ...] -And with one simple, cleverer XPath we are able to extract all quotes from -the page. We could have constructed a loop over our first XPath to increase -the number of the last ``div``, but this would have been unnecessarily +And with one simple, cleverer XPath we are able to extract all quotes from +the page. We could have constructed a loop over our first XPath to increase +the number of the last ``div``, but this would have been unnecessarily complex and by simply constructing an XPath with ``has-class("text")`` -we were able to extract all quotes in one line. +we were able to extract all quotes in one line. -The `Inspector` has a lot of other helpful features, such as searching in the +The `Inspector` has a lot of other helpful features, such as searching in the source code or directly scrolling to an element you selected. Let's demonstrate -a use case: +a use case: -Say you want to find the ``Next`` button on the page. Type ``Next`` into the -search bar on the top right of the `Inspector`. You should get two results. -The first is a ``li`` tag with the ``class="next"``, the second the text +Say you want to find the ``Next`` button on the page. Type ``Next`` into the +search bar on the top right of the `Inspector`. You should get two results. +The first is a ``li`` tag with the ``class="next"``, the second the text of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``. If you hover over the tag, you'll see the button highlighted. From here -we could easily create a :ref:`Link Extractor ` to -follow the pagination. On a simple site such as this, there may not be +we could easily create a :ref:`Link Extractor ` to +follow the pagination. On a simple site such as this, there may not be the need to find an element visually but the ``Scroll into View`` function -can be quite useful on complex sites. +can be quite useful on complex sites. Note that the search bar can also be used to search for and test CSS -selectors. For example, you could search for ``span.text`` to find -all quote texts. Instead of a full text search, this searches for -exactly the ``span`` tag with the ``class="text"`` in the page. +selectors. For example, you could search for ``span.text`` to find +all quote texts. Instead of a full text search, this searches for +exactly the ``span`` tag with the ``class="text"`` in the page. .. _topics-network-tool: The Network-tool ================ While scraping you may come across dynamic webpages where some parts -of the page are loaded dynamically through multiple requests. While -this can be quite tricky, the `Network`-tool in the Developer Tools +of the page are loaded dynamically through multiple requests. While +this can be quite tricky, the `Network`-tool in the Developer Tools greatly facilitates this task. To demonstrate the Network-tool, let's -take a look at the page `quotes.toscrape.com/scroll`_. +take a look at the page `quotes.toscrape.com/scroll`_. -The page is quite similar to the basic `quotes.toscrape.com`_-page, -but instead of the above-mentioned ``Next`` button, the page -automatically loads new quotes when you scroll to the bottom. We -could go ahead and try out different XPaths directly, but instead +The page is quite similar to the basic `quotes.toscrape.com`_-page, +but instead of the above-mentioned ``Next`` button, the page +automatically loads new quotes when you scroll to the bottom. We +could go ahead and try out different XPaths directly, but instead we'll check another quite useful command from the Scrapy shell: .. skip: next @@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell: (...) >>> view(response) -A browser window should open with the webpage but with one -crucial difference: Instead of the quotes we just see a greenish -bar with the word ``Loading...``. +A browser window should open with the webpage but with one +crucial difference: Instead of the quotes we just see a greenish +bar with the word ``Loading...``. .. image:: _images/network_01.png :width: 777 @@ -189,21 +189,21 @@ bar with the word ``Loading...``. :alt: Response from quotes.toscrape.com/scroll The ``view(response)`` command let's us view the response our -shell or later our spider receives from the server. Here we see -that some basic template is loaded which includes the title, +shell or later our spider receives from the server. Here we see +that some basic template is loaded which includes the title, the login-button and the footer, but the quotes are missing. This tells us that the quotes are being loaded from a different request -than ``quotes.toscrape/scroll``. +than ``quotes.toscrape/scroll``. -If you click on the ``Network`` tab, you will probably only see -two entries. The first thing we do is enable persistent logs by -clicking on ``Persist Logs``. If this option is disabled, the +If you click on the ``Network`` tab, you will probably only see +two entries. The first thing we do is enable persistent logs by +clicking on ``Persist Logs``. If this option is disabled, the log is automatically cleared each time you navigate to a different -page. Enabling this option is a good default, since it gives us -control on when to clear the logs. +page. Enabling this option is a good default, since it gives us +control on when to clear the logs. If we reload the page now, you'll see the log get populated with six -new requests. +new requests. .. image:: _images/network_02.png :width: 777 @@ -212,31 +212,31 @@ new requests. Here we see every request that has been made when reloading the page and can inspect each request and its response. So let's find out -where our quotes are coming from: +where our quotes are coming from: -First click on the request with the name ``scroll``. On the right +First click on the request with the name ``scroll``. On the right you can now inspect the request. In ``Headers`` you'll find details about the request headers, such as the URL, the method, the IP-address, and so on. We'll ignore the other tabs and click directly on ``Response``. -What you should see in the ``Preview`` pane is the rendered HTML-code, -that is exactly what we saw when we called ``view(response)`` in the -shell. Accordingly the ``type`` of the request in the log is ``html``. -The other requests have types like ``css`` or ``js``, but what -interests us is the one request called ``quotes?page=1`` with the -type ``json``. +What you should see in the ``Preview`` pane is the rendered HTML-code, +that is exactly what we saw when we called ``view(response)`` in the +shell. Accordingly the ``type`` of the request in the log is ``html``. +The other requests have types like ``css`` or ``js``, but what +interests us is the one request called ``quotes?page=1`` with the +type ``json``. -If we click on this request, we see that the request URL is +If we click on this request, we see that the request URL is ``http://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click -on the request and open ``Open in new tab`` to get a better overview. +on the request and open ``Open in new tab`` to get a better overview. .. image:: _images/network_03.png :width: 777 :height: 375 :alt: JSON-object returned from the quotes.toscrape API -With this response we can now easily parse the JSON-object and +With this response we can now easily parse the JSON-object and also request each page to get every quote on the site:: import scrapy @@ -255,17 +255,17 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page) + url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) -This spider starts at the first page of the quotes-API. With each -response, we parse the ``response.text`` and assign it to ``data``. -This lets us operate on the JSON-object like on a Python dictionary. +This spider starts at the first page of the quotes-API. With each +response, we parse the ``response.text`` and assign it to ``data``. +This lets us operate on the JSON-object like on a Python dictionary. We iterate through the ``quotes`` and print out the ``quote["text"]``. -If the handy ``has_next`` element is ``true`` (try loading +If the handy ``has_next`` element is ``true`` (try loading `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a -page-number greater than 10), we increment the ``page`` attribute -and ``yield`` a new request, inserting the incremented page-number +page-number greater than 10), we increment the ``page`` attribute +and ``yield`` a new request, inserting the incremented page-number into our ``url``. .. _requests-from-curl: @@ -289,11 +289,16 @@ request:: "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") Alternatively, if you want to know the arguments needed to recreate that -request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs` -function to get a dictionary with the equivalent arguments. +request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs` +function to get a dictionary with the equivalent arguments: + +.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs + +Note that to translate a cURL command into a Scrapy request, +you may use `curl2scrapy `_. As you can see, with a few inspections in the `Network`-tool we -were able to easily replicate the dynamic requests of the scrolling +were able to easily replicate the dynamic requests of the scrolling functionality of the page. Crawling dynamic pages can be quite daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 73648994d..b539c23df 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -202,6 +202,16 @@ CookiesMiddleware sends them back on subsequent requests (from that spider), just like web browsers do. + .. caution:: When non-UTF8 encoded byte sequences are passed to a + :class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log + a warning. Refer to :ref:`topics-logging-advanced-customization` + to customize the logging behaviour. + + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + The following settings can be used to configure the cookie middleware: * :setting:`COOKIES_ENABLED` @@ -212,8 +222,6 @@ The following settings can be used to configure the cookie middleware: Multiple cookie sessions per spider ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 0.15 - There is support for keeping multiple cookie sessions per spider by using the :reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar (session), but you can pass an identifier to use different ones. @@ -470,8 +478,6 @@ DBM storage backend .. class:: DbmCacheStorage - .. versionadded:: 0.13 - A DBM_ storage backend is also available for the HTTP cache middleware. By default, it uses the :mod:`dbm`, but you can change it with the @@ -544,15 +550,10 @@ settings: HTTPCACHE_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.11 - Default: ``False`` Whether the HTTP cache will be enabled. -.. versionchanged:: 0.11 - Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache. - .. setting:: HTTPCACHE_EXPIRATION_SECS HTTPCACHE_EXPIRATION_SECS @@ -565,9 +566,6 @@ Expiration time for cached requests, in seconds. Cached requests older than this time will be re-downloaded. If zero, cached requests will never expire. -.. versionchanged:: 0.11 - Before 0.11, zero meant cached requests always expire. - .. setting:: HTTPCACHE_DIR HTTPCACHE_DIR @@ -584,8 +582,6 @@ project data dir. For more info see: :ref:`topics-project-structure`. HTTPCACHE_IGNORE_HTTP_CODES ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``[]`` Don't cache response with these HTTP codes. @@ -604,8 +600,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade HTTPCACHE_IGNORE_SCHEMES ^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``['file']`` Don't cache responses with these URI schemes. @@ -624,8 +618,6 @@ The class which implements the cache storage backend. HTTPCACHE_DBM_MODULE ^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``'dbm'`` The database module to use in the :ref:`DBM storage backend @@ -636,8 +628,6 @@ The database module to use in the :ref:`DBM storage backend HTTPCACHE_POLICY ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.18 - Default: ``'scrapy.extensions.httpcache.DummyPolicy'`` The class which implements the cache policy. @@ -647,8 +637,6 @@ The class which implements the cache policy. HTTPCACHE_GZIP ^^^^^^^^^^^^^^ -.. versionadded:: 1.0 - Default: ``False`` If enabled, will compress all cached data with gzip. @@ -659,8 +647,6 @@ This setting is specific to the Filesystem backend. HTTPCACHE_ALWAYS_STORE ^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``False`` If enabled, will cache pages unconditionally. @@ -679,8 +665,6 @@ responses you feed to the cache middleware. HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``[]`` List of Cache-Control directives in responses to be ignored. @@ -705,11 +689,14 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ responses, - provided `brotlipy`_ is installed. + This middleware also supports decoding `brotli-compressed`_ as well as + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is + installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ +.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt +.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -730,8 +717,6 @@ HttpProxyMiddleware .. module:: scrapy.downloadermiddlewares.httpproxy :synopsis: Http Proxy Middleware -.. versionadded:: 0.8 - .. reqmeta:: proxy .. class:: HttpProxyMiddleware @@ -739,7 +724,7 @@ HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the ``proxy`` meta value for :class:`~scrapy.http.Request` objects. - Like the Python standard library modules `urllib`_ and `urllib2`_, it obeys + Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: * ``http_proxy`` @@ -751,9 +736,6 @@ HttpProxyMiddleware Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. -.. _urllib: https://docs.python.org/2/library/urllib.html -.. _urllib2: https://docs.python.org/2/library/urllib2.html - RedirectMiddleware ------------------ @@ -815,8 +797,6 @@ RedirectMiddleware settings REDIRECT_ENABLED ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Redirect middleware will be enabled. @@ -829,6 +809,7 @@ REDIRECT_MAX_TIMES Default: ``20`` The maximum number of redirections that will be followed for a single request. +After this maximum, the request's response is returned as is. MetaRefreshMiddleware --------------------- @@ -857,8 +838,6 @@ MetaRefreshMiddleware settings METAREFRESH_ENABLED ^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.17 - Default: ``True`` Whether the Meta Refresh middleware will be enabled. @@ -913,6 +892,11 @@ settings (see the settings documentation for more info): If :attr:`Request.meta ` has ``dont_retry`` key set to True, the request will be ignored by this middleware. +To retry requests from a spider callback, you can use the +:func:`get_retry_request` function: + +.. autofunction:: get_retry_request + RetryMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -921,8 +905,6 @@ RetryMiddleware Settings RETRY_ENABLED ^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Retry middleware will be enabled. @@ -955,6 +937,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_PRIORITY_ADJUST + +RETRY_PRIORITY_ADJUST +--------------------- + +Default: ``-1`` + +Adjust retry request priority relative to original request: + +- a positive priority adjust means higher priority. +- **a negative priority adjust (default) means lower priority.** + .. _topics-dlmw-robots: @@ -1036,8 +1030,7 @@ Scrapy uses this parser by default. RobotFileParser ~~~~~~~~~~~~~~~ -Based on `RobotFileParser -`_: +Based on :class:`~urllib.robotparser.RobotFileParser`: * is Python's built-in robots.txt_ parser @@ -1177,8 +1170,6 @@ AjaxCrawlMiddleware Settings AJAXCRAWL_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.21 - Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. You may want to diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index b98133676..495111b56 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -104,6 +104,9 @@ If you get the expected response `sometimes`, but not always, the issue is probably not your request, but the target server. The target server might be buggy, overloaded, or :ref:`banning ` some of your requests. +Note that to translate a cURL command into a Scrapy request, +you may use `curl2scrapy `_. + .. _topics-handling-response-formats: Handling different response formats @@ -115,7 +118,7 @@ data from it depends on the type of response: - If the response is HTML or XML, use :ref:`selectors ` as usual. -- If the response is JSON, use `json.loads`_ to load the desired data from +- If the response is JSON, use :func:`json.loads` to load the desired data from :attr:`response.text `:: data = json.loads(response.text) @@ -130,8 +133,9 @@ data from it depends on the type of response: - If the response is JavaScript, or HTML with a ``