diff --git a/.bandit.yml b/.bandit.yml index 243379b0b..41f1bb597 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -8,6 +8,7 @@ skips: - B311 - B320 - B321 +- B324 - B402 # https://github.com/scrapy/scrapy/issues/4180 - B403 - B404 diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 956c512cb..1d9b9c02f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.1 +current_version = 2.6.1 commit = True tag = True tag_name = {new_version} diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..1c503fb0b --- /dev/null +++ b/.flake8 @@ -0,0 +1,19 @@ +[flake8] + +max-line-length = 119 +ignore = W503 + +exclude = +# Exclude files that are meant to provide top-level imports +# E402: Module level import not at top of file +# F401: Module imported but unused + scrapy/__init__.py E402 + scrapy/core/downloader/handlers/http.py F401 + scrapy/http/__init__.py F401 + scrapy/linkextractors/__init__.py E402 F401 + scrapy/selector/__init__.py F401 + scrapy/spiders/__init__.py E402 F401 + + # Issues pending a review: + scrapy/utils/url.py F403 F405 + tests/test_loader.py E741 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 000000000..b26f344ff --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,41 @@ +name: Checks +on: [push, pull_request] + +jobs: + checks: + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + include: + - python-version: "3.10" + env: + TOXENV: security + - python-version: "3.10" + env: + TOXENV: flake8 + # Pylint requires installing reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 + - python-version: 3.8 + env: + TOXENV: pylint + - python-version: 3.7 + env: + TOXENV: typing + - python-version: "3.10" # Keep in sync with .readthedocs.yml + env: + TOXENV: docs + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run check + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 28771216c..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Run test suite -on: [push, pull_request] - -jobs: - test-windows: - name: "Windows Tests" - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-latest] - python-version: [3.7, 3.8] - env: [TOXENV: py] - include: - - os: windows-latest - python-version: 3.6 - env: - TOXENV: windows-pinned - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Run test suite - env: ${{ matrix.env }} - run: | - pip install -U tox twine wheel codecov - tox diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..44b682830 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,31 @@ +name: Publish +on: [push] + +jobs: + publish: + runs-on: ubuntu-18.04 + if: startsWith(github.event.ref, 'refs/tags/') + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + + - name: Check Tag + id: check-release-tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then + echo ::set-output name=release_tag::true + fi + + - name: Publish to PyPI + if: steps.check-release-tag.outputs.release_tag == 'true' + run: | + pip install --upgrade setuptools wheel twine + python setup.py sdist bdist_wheel + export TWINE_USERNAME=__token__ + export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} + twine upload dist/* diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml new file mode 100644 index 000000000..7819a4e12 --- /dev/null +++ b/.github/workflows/tests-macos.yml @@ -0,0 +1,26 @@ +name: macOS +on: [push, pull_request] + +jobs: + tests: + runs-on: macos-10.15 + strategy: + fail-fast: false + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + run: | + pip install -U tox + tox -e py + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml new file mode 100644 index 000000000..be40c7c71 --- /dev/null +++ b/.github/workflows/tests-ubuntu.yml @@ -0,0 +1,76 @@ +name: Ubuntu +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + include: + - python-version: 3.8 + env: + TOXENV: py + - python-version: 3.9 + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio + - python-version: pypy3 + env: + TOXENV: pypy3 + PYPY_VERSION: 3.9-v7.3.9 + + # pinned deps + - python-version: 3.7.13 + env: + TOXENV: pinned + - python-version: 3.7.13 + env: + TOXENV: asyncio-pinned + - python-version: pypy3 + env: + TOXENV: pypy3-pinned + PYPY_VERSION: 3.7-v7.3.5 + + # extras + # extra-deps includes reppy, which does not support Python 3.9 + # https://github.com/seomoz/reppy/issues/122 + - python-version: 3.8 + env: + TOXENV: extra-deps + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system libraries + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4' + run: | + sudo apt-get update + # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version + sudo apt-get install libxml2-dev/bionic-updates libxslt-dev + + - name: Run tests + env: ${{ matrix.env }} + run: | + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + pip install -U tox + tox + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml new file mode 100644 index 000000000..955b9b449 --- /dev/null +++ b/.github/workflows/tests-windows.yml @@ -0,0 +1,39 @@ +name: Windows +on: [push, pull_request] + +jobs: + tests: + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - python-version: 3.7 + env: + TOXENV: windows-pinned + - python-version: 3.8 + env: + TOXENV: py + - python-version: 3.9 + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + env: ${{ matrix.env }} + run: | + pip install -U tox + tox diff --git a/.gitignore b/.gitignore index 795e2605e..d77d24624 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ htmlcov/ .coverage .pytest_cache/ .coverage.* +coverage.* +test-output.* .cache/ .mypy_cache/ /tests/keys/localhost.crt diff --git a/.readthedocs.yml b/.readthedocs.yml index e4d3f02cc..390be3749 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,10 +3,15 @@ formats: all sphinx: configuration: docs/conf.py fail_on_warning: true + +build: + os: ubuntu-20.04 + tools: + # For available versions, see: + # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python + python: "3.10" # Keep in sync with .github/workflows/checks.yml + python: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image - version: 3.7 # Keep in sync with .travis.yml install: - requirements: docs/requirements.txt - path: . diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b883c5b78..000000000 --- a/.travis.yml +++ /dev/null @@ -1,75 +0,0 @@ -language: python -dist: xenial -branches: - only: - - master - - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ -matrix: - include: - - env: TOXENV=security - python: 3.8 - - env: TOXENV=flake8 - python: 3.8 - - env: TOXENV=pylint - python: 3.8 - - env: TOXENV=docs - python: 3.7 # Keep in sync with .readthedocs.yml - - env: TOXENV=typing - python: 3.8 - - - env: TOXENV=pinned - python: 3.6.1 - - env: TOXENV=asyncio-pinned - python: 3.6.1 - - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - - - env: TOXENV=py - python: 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 - - - env: TOXENV=py - python: 3.7 - - - env: TOXENV=py PYPI_RELEASE_JOB=true - python: 3.8 - dist: bionic - - env: TOXENV=extra-deps - python: 3.8 - dist: bionic - - env: TOXENV=asyncio - python: 3.8 - dist: bionic -install: - - | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi - - pip install -U tox twine wheel codecov - -script: tox -after_success: - - codecov -notifications: - irc: - use_notice: true - skip_join: true - channels: - - irc.freenode.org#scrapy -cache: - directories: - - $HOME/.cache/pip -deploy: - provider: pypi - distributions: "sdist bdist_wheel" - user: scrapy - password: - secure: JaAKcy1AXWXDK3LXdjOtKyaVPCSFoCGCnW15g4f65E/8Fsi9ZzDfmBa4Equs3IQb/vs/if2SVrzJSr7arN7r9Z38Iv1mUXHkFAyA3Ym8mThfABBzzcUWEQhIHrCX0Tdlx9wQkkhs+PZhorlmRS4gg5s6DzPaeA2g8SCgmlRmFfA= - on: - tags: true - repo: scrapy/scrapy - condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" diff --git a/AUTHORS b/AUTHORS index bcaa1ecd3..9706adf42 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,8 +1,8 @@ Scrapy was brought to life by Shane Evans while hacking a scraping framework prototype for Mydeco (mydeco.com). It soon became maintained, extended and improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to -bootstrap the project. In mid-2011, Scrapinghub became the new official -maintainer. +bootstrap the project. In mid-2011, Scrapinghub (now Zyte) became the new +official maintainer. Here is the list of the primary authors & contributors: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d1cd3e517..902cd523e 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -55,7 +55,7 @@ further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@scrapinghub.com. All +reported by contacting the project team at opensource@zyte.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. @@ -72,3 +72,6 @@ available at [http://contributor-covenant.org/version/1/4][version]. [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/README.rst b/README.rst index a8f2ba52b..b543a30f4 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,5 @@ +.. image:: https://scrapy.org/img/scrapylogo.png + ====== Scrapy ====== @@ -10,9 +12,17 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg - :target: https://travis-ci.org/scrapy/scrapy - :alt: Build Status +.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu + :alt: Ubuntu + +.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS + +.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows + :alt: Windows .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg :target: https://pypi.python.org/pypi/Scrapy @@ -34,13 +44,20 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. +Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other +contributors`_. + +.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Zyte: https://www.zyte.com/ + Check the Scrapy homepage at https://scrapy.org for more information, including a list of features. + Requirements ============ -* Python 3.6+ +* Python 3.7+ * Works on Linux, Windows, macOS, BSD Install @@ -81,7 +98,7 @@ Please note that this project is released with a Contributor Code of Conduct (see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@scrapinghub.com. +Please report unacceptable behavior to opensource@zyte.com. Companies using Scrapy ====================== diff --git a/conftest.py b/conftest.py index 68b855c08..117087790 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,9 @@ from pathlib import Path import pytest +from twisted.web.http import H2_ENABLED + +from scrapy.utils.reactor import install_reactor from tests.keys import generate_keys @@ -18,10 +21,19 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -for line in open('tests/ignores.txt'): - file_path = line.strip() - if file_path and file_path[0] != '#': - collect_ignore.append(file_path) +with open('tests/ignores.txt') as reader: + for line in reader: + file_path = line.strip() + if file_path and file_path[0] != '#': + collect_ignore.append(file_path) + +if not H2_ENABLED: + collect_ignore.extend( + ( + 'scrapy/core/downloader/handlers/http2.py', + *_py_files("scrapy/core/http2"), + ) + ) @pytest.fixture() @@ -40,6 +52,14 @@ def pytest_collection_modifyitems(session, config, items): pass +def pytest_addoption(parser): + parser.addoption( + "--reactor", + default="default", + choices=["default", "asyncio"], + ) + + @pytest.fixture(scope='class') def reactor_pytest(request): if not request.cls: @@ -55,5 +75,16 @@ def only_asyncio(request, reactor_pytest): pytest.skip('This test is only run with --reactor=asyncio') +@pytest.fixture(autouse=True) +def only_not_asyncio(request, reactor_pytest): + if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio': + pytest.skip('This test is only run without --reactor=asyncio') + + +def pytest_configure(config): + if config.getoption("--reactor") == "asyncio": + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + + # Generate localhost certificate files, needed by some tests generate_keys() diff --git a/docs/Makefile b/docs/Makefile index ff68bf1ae..87d5d3047 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,7 +8,7 @@ PYTHON = python SPHINXOPTS = PAPER = SOURCES = -SHELL = /bin/bash +SHELL = /usr/bin/env bash ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ -D latex_elements.papersize=$(PAPER) \ diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 000000000..64f16939c --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,10 @@ +/* Move lists closer to their introducing paragraph */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} +.rst-content p + ol, .rst-content p + ul { + margin-top: -18px; /* Compensates margin-top: 24px of p */ +} +.rst-content dl p + ol, .rst-content dl p + ul { + margin-top: -6px; /* Compensates margin-top: 12px of p */ +} \ No newline at end of file diff --git a/docs/_static/selectors-sample1.html b/docs/_static/selectors-sample1.html index 8a79a3381..915718832 100644 --- a/docs/_static/selectors-sample1.html +++ b/docs/_static/selectors-sample1.html @@ -1,16 +1,17 @@ - - - - Example website - - -
- Name: My image 1
- Name: My image 2
- Name: My image 3
- Name: My image 4
- Name: My image 5
-
- - + + + + + Example website + + +
+ Name: My image 1
image1 + Name: My image 2
image2 + Name: My image 3
image3 + Name: My image 4
image4 + Name: My image 5
image5 +
+ + \ No newline at end of file diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index a6f6cbda8..18a5231ee 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -3,14 +3,9 @@ {% block footer %} {{ super() }} {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index 27d2b5dff..378b01804 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -122,7 +122,6 @@ html_theme = 'sphinx_rtd_theme' import sphinx_rtd_theme html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # The style sheet to use for HTML and HTML Help pages. A file of that name # must exist either in Sphinx' static/ path, or in one of the custom paths # given in html_static_path. @@ -183,6 +182,10 @@ html_copy_source = True # Output file base name for HTML help builder. htmlhelp_basename = 'Scrapydoc' +html_css_files = [ + 'custom.css', +] + # Options for LaTeX output # ------------------------ @@ -269,7 +272,6 @@ coverage_ignore_pyobjects = [ r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions # Never documented before, and deprecated now. - r'^scrapy\.item\.DictItem$', r'^scrapy\.linkextractors\.FilteringLinkExtractor$', # Implementation detail of LxmlLinkExtractor @@ -283,6 +285,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { 'attrs': ('https://www.attrs.org/en/stable/', None), 'coverage': ('https://coverage.readthedocs.io/en/stable', None), + 'cryptography' : ('https://cryptography.io/en/latest/', None), 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), 'pytest': ('https://docs.pytest.org/en/latest', None), @@ -291,7 +294,9 @@ intersphinx_mapping = { 'tox': ('https://tox.readthedocs.io/en/latest', None), 'twisted': ('https://twistedmatrix.com/documents/current', None), 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None), + 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None), } +intersphinx_disabled_reftypes = [] # Options for sphinx-hoverxref options @@ -300,10 +305,14 @@ intersphinx_mapping = { hoverxref_auto_ref = True hoverxref_role_types = { "class": "tooltip", + "command": "tooltip", "confval": "tooltip", "hoverxref": "tooltip", "mod": "tooltip", "ref": "tooltip", + "reqmeta": "tooltip", + "setting": "tooltip", + "signal": "tooltip", } hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal'] diff --git a/docs/conftest.py b/docs/conftest.py index 8c735e838..a0636f8ac 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -3,7 +3,11 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE from scrapy.http.response.html import HtmlResponse from sybil import Sybil -from sybil.parsers.codeblock import CodeBlockParser +try: + # >2.0.1 + from sybil.parsers.codeblock import PythonCodeBlockParser +except ImportError: + from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser from sybil.parsers.doctest import DocTestParser from sybil.parsers.skip import skip @@ -21,7 +25,7 @@ def setup(namespace): pytest_collect_file = Sybil( parsers=[ DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE), - CodeBlockParser(future_imports=['print_function']), + PythonCodeBlockParser(future_imports=['print_function']), skip, ], pattern='*.rst', diff --git a/docs/contributing.rst b/docs/contributing.rst index 4d2580a6c..946bdc23e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use: To run the tests on a specific :doc:`tox ` environment, use ``-e `` with an environment name from ``tox.ini``. For example, to run -the tests with Python 3.6 use:: +the tests with Python 3.7 use:: - tox -e py36 + tox -e py37 You can also specify a comma-separated list of environments, and use :ref:`tox’s parallel mode ` to run the tests on multiple environments in parallel:: - tox -e py36,py38 -p auto + tox -e py37,py38 -p auto To pass command-line options to :doc:`pytest `, add them after ``--`` in your call to :doc:`tox `. Using ``--`` overrides the @@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.6 :doc:`tox ` environment using all your CPU cores:: +the Python 3.7 :doc:`tox ` environment using all your CPU cores:: - tox -e py36 -- scrapy tests -n auto + tox -e py37 -- scrapy tests -n auto To see coverage report install :doc:`coverage ` (``pip install coverage``) and run: diff --git a/docs/faq.rst b/docs/faq.rst index 9346ec358..8a9ba809b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -94,15 +94,6 @@ How can I scrape an item with attributes in different pages? See :ref:`topics-request-response-ref-request-callback-arguments`. - -Scrapy crashes with: ImportError: No module named win32api ----------------------------------------------------------- - -You need to install `pywin32`_ because of `this Twisted bug`_. - -.. _pywin32: https://sourceforge.net/projects/pywin32/ -.. _this Twisted bug: https://twistedmatrix.com/trac/ticket/3707 - How can I simulate a user login in my spider? --------------------------------------------- @@ -145,6 +136,41 @@ How can I make Scrapy consume less memory? See previous question. +How can I prevent memory errors due to many allowed domains? +------------------------------------------------------------ + +If you have a spider with a long list of +:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider +replacing the default +:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware +with a :ref:`custom spider middleware ` that requires +less memory. For example: + +- If your domain names are similar enough, use your own regular expression + instead joining the strings in + :attr:`~scrapy.Spider.allowed_domains` into a complex regular + expression. + +- If you can `meet the installation requirements`_, use pyre2_ instead of + Python’s re_ to compile your URL-filtering regular expression. See + :issue:`1908`. + +See also other suggestions at `StackOverflow`_. + +.. note:: Remember to disable + :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable + your custom implementation:: + + SPIDER_MIDDLEWARES = { + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, + 'myproject.middlewares.CustomOffsiteMiddleware': 500, + } + +.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation +.. _pyre2: https://github.com/andreasvc/pyre2 +.. _re: https://docs.python.org/library/re.html +.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 + Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- @@ -315,6 +341,7 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. + .. _faq-split-item: How to split an item into multiple items in an item pipeline? @@ -363,15 +390,27 @@ How can I cancel the download of a given response? -------------------------------------------------- In some situations, it might be useful to stop the download of a certain response. -For instance, if you only need the first part of a large response and you would like -to save resources by avoiding the download of the whole body. -In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received` -signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to -the :ref:`topics-stop-response-download` topic for additional information and examples. +For instance, sometimes you can determine whether or not you need the full contents +of a response by inspecting its headers or the first bytes of its body. In that case, +you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received` +or :class:`~scrapy.signals.headers_received` signals and raising a +:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the +:ref:`topics-stop-response-download` topic for additional information and examples. + + +Running ``runspider`` I get ``error: No spider found in file: `` +-------------------------------------------------------------------------- + +This may happen if your Scrapy project has a spider module with a name that +conflicts with the name of one of the `Python standard library modules`_, such +as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed. +See :issue:`2680`. .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 +.. _Python standard library modules: https://docs.python.org/py-modindex.html +.. _Python package: https://pypi.org/ .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search +.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index da264fb34..75e08f537 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,8 @@ testing. .. _web crawling: https://en.wikipedia.org/wiki/Web_crawler .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping +.. _getting-help: + Getting help ============ @@ -24,12 +26,14 @@ Having trouble? We'd like to help! * Search for questions on the archives of the `scrapy-users mailing list`_. * Ask a question in the `#scrapy IRC channel`_, * Report bugs with Scrapy in our `issue tracker`_. +* Join the Discord community `Scrapy Discord`_. .. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users .. _Scrapy subreddit: https://www.reddit.com/r/scrapy/ .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues +.. _Scrapy Discord: https://discord.gg/mv3yErfpvq First steps @@ -227,6 +231,7 @@ Extending Scrapy topics/extensions topics/api topics/signals + topics/scheduler topics/exporters @@ -248,6 +253,9 @@ Extending Scrapy :doc:`topics/signals` See all available signals and how to work with them. +:doc:`topics/scheduler` + Understand the scheduler component. + :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). diff --git a/docs/intro/examples.rst b/docs/intro/examples.rst index 96363c7d5..edff894c6 100644 --- a/docs/intro/examples.rst +++ b/docs/intro/examples.rst @@ -7,7 +7,7 @@ Examples The best way to learn is with examples, and Scrapy is no exception. For this reason, there is an example Scrapy project named quotesbot_, that you can use to play and learn more about Scrapy. It contains two spiders for -http://quotes.toscrape.com, one using CSS selectors and another one using XPath +https://quotes.toscrape.com, one using CSS selectors and another one using XPath expressions. The quotesbot_ project is available at: https://github.com/scrapy/quotesbot. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 3bfd3bc3b..80a9c16d6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,9 +9,10 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.6+, either the CPython implementation (default) or -the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). +Scrapy requires Python 3.7+, either the CPython implementation (default) or +the PyPy 7.3.5+ implementation (see :ref:`python:implementations`). +.. _intro-install-scrapy: Installing Scrapy ================= @@ -29,13 +30,13 @@ you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, +to avoid conflicting with your system packages. + Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, -to avoid conflicting with your system packages. - For more detailed and platform specifics instructions, as well as troubleshooting information, read on. @@ -51,16 +52,6 @@ Scrapy is written in pure Python and depends on a few key Python packages (among * `twisted`_, an asynchronous networking framework * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs -The minimal versions which Scrapy is tested against are: - -* Twisted 14.0 -* lxml 3.4 -* pyOpenSSL 0.14 - -Scrapy may work with older versions of these packages -but it is not guaranteed it will continue working -because it’s not being tested against them. - Some of these packages themselves depends on non-Python packages that might require additional installation steps depending on your platform. Please check :ref:`platform-specific guides below `. @@ -69,10 +60,9 @@ In case of any trouble related to these dependencies, please refer to their respective installation instructions: * `lxml installation`_ -* `cryptography installation`_ +* :doc:`cryptography installation ` .. _lxml installation: https://lxml.de/installation.html -.. _cryptography installation: https://cryptography.io/en/latest/installation/ .. _intro-using-virtualenv: @@ -118,6 +108,27 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: conda install -c conda-forge scrapy +To install Scrapy on Windows using ``pip``: + +.. warning:: + This installation method requires “Microsoft Visual C++” for installing some + Scrapy dependencies, which demands significantly more disk space than Anaconda. + +#. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. + +#. Run the Visual Studio Installer. + +#. Under the Workloads section, select **C++ build tools**. + +#. Check the installation details and make sure following packages are selected as optional components: + + * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) + + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) + +#. Install the Visual Studio Build Tools. + +Now, you should be able to :ref:`install Scrapy ` using ``pip``. .. _intro-install-ubuntu: @@ -169,7 +180,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to successfully install Scrapy and its dependencies. Here are some proposed solutions: -* *(Recommended)* **Don't** use system python, install a new, updated version +* *(Recommended)* **Don't** use system Python. Install a new, updated version that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: @@ -213,8 +224,8 @@ For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. This means that these dependencies will be built during installation. -On macOS, you are likely to face an issue with building Cryptography dependency, -solution to this problem is described +On macOS, you are likely to face an issue with building the Cryptography +dependency. The solution to this problem is described `here `_, that is to ``brew install openssl`` and then export the flags that this command recommends (only needed when installing Scrapy). Installing on Linux has no special @@ -265,10 +276,10 @@ For details, see `Issue #2473 `_. .. _cryptography: https://cryptography.io/en/latest/ .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ .. _setuptools: https://pypi.python.org/pypi/setuptools -.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Scrapinghub: https://scrapinghub.com .. _Anaconda: https://docs.anaconda.com/anaconda/ .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html +.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio +.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index dd80c7bd0..f3d652621 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -4,7 +4,7 @@ Scrapy at a glance ================== -Scrapy is an application framework for crawling web sites and extracting +Scrapy (/ˈskreɪpaɪ/) is an application framework for crawling web sites and extracting structured data which can be used for a wide range of useful applications, like data mining, information processing or historical archival. @@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. Here's the code for a spider that scrapes famous quotes from website -http://quotes.toscrape.com, following the pagination:: +https://quotes.toscrape.com, following the pagination:: import scrapy @@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination:: class QuotesSpider(scrapy.Spider): name = 'quotes' start_urls = [ - 'http://quotes.toscrape.com/tag/humor/', + 'https://quotes.toscrape.com/tag/humor/', ] def parse(self, response): diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 9270ff42c..cde1b1ef4 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -7,7 +7,7 @@ Scrapy Tutorial In this tutorial, we'll assume that Scrapy is already installed on your system. If that's not the case, see :ref:`intro-install`. -We are going to scrape `quotes.toscrape.com `_, a website +We are going to scrape `quotes.toscrape.com `_, a website that lists quotes from famous authors. This tutorial will walk you through these tasks: @@ -78,7 +78,7 @@ Our first Spider Spiders are classes that you define and that Scrapy uses to scrape information from a website (or a group of websites). They must subclass -:class:`~scrapy.spiders.Spider` and define the initial requests to make, +:class:`~scrapy.Spider` and define the initial requests to make, optionally how to follow links in the pages, and how to parse the downloaded page content to extract data. @@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named def start_requests(self): urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -107,26 +107,26 @@ This is the code for our first Spider. Save it in a file named self.log(f'Saved file {filename}') -As you can see, our Spider subclasses :class:`scrapy.Spider ` +As you can see, our Spider subclasses :class:`scrapy.Spider ` and defines some attributes and methods: -* :attr:`~scrapy.spiders.Spider.name`: identifies the Spider. It must be +* :attr:`~scrapy.Spider.name`: identifies the Spider. It must be unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.spiders.Spider.start_requests`: must return an iterable of +* :meth:`~scrapy.Spider.start_requests`: must return an iterable of Requests (you can return a list of requests or write a generator function) which the Spider will begin to crawl from. Subsequent requests will be generated successively from these initial requests. -* :meth:`~scrapy.spiders.Spider.parse`: a method that will be called to handle +* :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter is an instance of :class:`~scrapy.http.TextResponse` that holds the page content and has further helpful methods to handle it. - The :meth:`~scrapy.spiders.Spider.parse` method usually parses the response, extracting + The :meth:`~scrapy.Spider.parse` method usually parses the response, extracting the scraped data as dicts and also finding new URLs to - follow and creating new requests (:class:`~scrapy.http.Request`) from them. + follow and creating new requests (:class:`~scrapy.Request`) from them. How to run our spider --------------------- @@ -143,9 +143,9 @@ similar to this:: 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened 2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) 2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023 - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished) @@ -162,7 +162,7 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects +Scrapy schedules the :class:`scrapy.Request ` objects returned by the ``start_requests`` method of the Spider. Upon receiving a response for each one, it instantiates :class:`~scrapy.http.Response` objects and calls the callback method associated with the request (in this case, the @@ -171,11 +171,11 @@ and calls the callback method associated with the request (in this case, the A shortcut to the start_requests method --------------------------------------- -Instead of implementing a :meth:`~scrapy.spiders.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.spiders.Spider.start_urls` class attribute +Instead of implementing a :meth:`~scrapy.Spider.start_requests` method +that generates :class:`scrapy.Request ` objects from URLs, +you can just define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.spiders.Spider.start_requests` to create the initial requests +of :meth:`~scrapy.Spider.start_requests` to create the initial requests for your spider:: import scrapy @@ -184,8 +184,8 @@ for your spider:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -194,9 +194,9 @@ for your spider:: with open(filename, 'wb') as f: f.write(response.body) -The :meth:`~scrapy.spiders.Spider.parse` method will be called to handle each +The :meth:`~scrapy.Spider.parse` method will be called to handle each of the requests for those URLs, even though we haven't explicitly told Scrapy -to do so. This happens because :meth:`~scrapy.spiders.Spider.parse` is Scrapy's +to do so. This happens because :meth:`~scrapy.Spider.parse` is Scrapy's default callback method, which is called for requests without an explicitly assigned callback. @@ -207,7 +207,7 @@ Extracting data The best way to learn how to extract data with Scrapy is trying selectors using the :ref:`Scrapy shell `. Run:: - scrapy shell 'http://quotes.toscrape.com/page/1/' + scrapy shell 'https://quotes.toscrape.com/page/1/' .. note:: @@ -217,18 +217,18 @@ using the :ref:`Scrapy shell `. Run:: On Windows, use double quotes instead:: - scrapy shell "http://quotes.toscrape.com/page/1/" + scrapy shell "https://quotes.toscrape.com/page/1/" You will see something like:: [ ... Scrapy log here ... ] - 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler [s] item {} - [s] request - [s] response <200 http://quotes.toscrape.com/page/1/> + [s] request + [s] response <200 https://quotes.toscrape.com/page/1/> [s] settings [s] spider [s] Useful shortcuts: @@ -241,14 +241,14 @@ object: .. invisible-code-block: python - response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html') + response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html') >>> response.css('title') [] The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of -:class:`~scrapy.selector.Selector` objects that wrap around XML/HTML elements +:class:`~scrapy.Selector` objects that wrap around XML/HTML elements and allow you to run further queries to fine-grain the selection or extract the data. @@ -277,9 +277,19 @@ As an alternative, you could've written: >>> response.css('title::text')[0].get() 'Quotes to Scrape' -However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList` -instance avoids an ``IndexError`` and returns ``None`` when it doesn't -find any element matching the selection. +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +raise an :exc:`IndexError` exception if there are no results:: + + >>> response.css('noelement')[0].get() + Traceback (most recent call last): + ... + IndexError: list index out of range + +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +if there are no results:: + +>>> response.css("noelement").get() There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail @@ -345,7 +355,7 @@ Extracting quotes and authors Now that you know a bit about selection and extraction, let's complete our spider by writing the code to extract the quotes from the web page. -Each quote in http://quotes.toscrape.com is represented by HTML elements that look +Each quote in https://quotes.toscrape.com is represented by HTML elements that look like this: .. code-block:: html @@ -369,7 +379,7 @@ like this: Let's open up scrapy shell and play a bit to find out how to extract the data we want:: - $ scrapy shell 'http://quotes.toscrape.com' + $ scrapy shell 'https://quotes.toscrape.com' We get a list of selectors for the quote HTML elements with: @@ -434,8 +444,8 @@ in the callback, as you can see below:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -448,9 +458,9 @@ in the callback, as you can see below:: If you run this spider, it will output the extracted data with the log:: - 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} - 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"} @@ -464,7 +474,7 @@ The simplest way to store the scraped data is by using :ref:`Feed exports scrapy crawl quotes -O quotes.json -That will generate an ``quotes.json`` file containing all scraped items, +That will generate a ``quotes.json`` file containing all scraped items, serialized in `JSON`_. The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead @@ -478,7 +488,7 @@ The `JSON Lines`_ format is useful because it's stream-like, you can easily append new records to it. It doesn't have the same problem of JSON when you run twice. Also, as each record is a separate line, you can process big files without having to fit everything in memory, there are tools like `JQ`_ to help -doing that at the command-line. +do that at the command-line. In small projects (like the one in this tutorial), that should be enough. However, if you want to perform more complex things with the scraped items, you @@ -495,7 +505,7 @@ Following links =============== Let's say, instead of just scraping the stuff from the first two pages -from http://quotes.toscrape.com, you want quotes from all the pages in the website. +from https://quotes.toscrape.com, you want quotes from all the pages in the website. Now that you know how to extract data from pages, let's see how to follow links from them. @@ -539,7 +549,7 @@ page, extracting data from it:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/1/', ] def parse(self, response): @@ -590,7 +600,7 @@ As a shortcut for creating Request objects you can use class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/1/', ] def parse(self, response): @@ -644,7 +654,7 @@ this time for scraping author information:: class AuthorSpider(scrapy.Spider): name = 'author' - start_urls = ['http://quotes.toscrape.com/'] + start_urls = ['https://quotes.toscrape.com/'] def parse(self, response): author_page_links = response.css('.author + a') @@ -670,7 +680,7 @@ the pagination links with the ``parse`` callback as we saw before. Here we're passing callbacks to :meth:`response.follow_all ` as positional arguments to make the code shorter; it also works for -:class:`~scrapy.http.Request`. +:class:`~scrapy.Request`. The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. @@ -717,7 +727,7 @@ with a specific tag, building the URL based on the argument:: name = "quotes" def start_requests(self): - url = 'http://quotes.toscrape.com/' + url = 'https://quotes.toscrape.com/' tag = getattr(self, 'tag', None) if tag is not None: url = url + 'tag/' + tag @@ -737,7 +747,7 @@ with a specific tag, building the URL based on the argument:: If you pass the ``tag=humor`` argument to this spider, you'll notice that it will only visit URLs from the ``humor`` tag, such as -``http://quotes.toscrape.com/tag/humor``. +``https://quotes.toscrape.com/tag/humor``. You can :ref:`learn more about handling spider arguments here `. diff --git a/docs/news.rst b/docs/news.rst index 0391506c4..2d0ab485e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,634 @@ Release notes ============= +.. _release-2.6.1: + +Scrapy 2.6.1 (2022-03-01) +------------------------- + +Fixes a regression introduced in 2.6.0 that would unset the request method when +following redirects. + + +.. _release-2.6.0: + +Scrapy 2.6.0 (2022-03-01) +------------------------- + +Highlights: + +* :ref:`Security fixes for cookie handling <2.6-security-fixes>` + +* Python 3.10 support + +* :ref:`asyncio support ` is no longer considered + experimental, and works out-of-the-box on Windows regardless of your Python + version + +* Feed exports now support :class:`pathlib.Path` output paths and per-feed + :ref:`item filtering ` and + :ref:`post-processing ` + +.. _2.6-security-fixes: + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- When a :class:`~scrapy.http.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.http.Request` object to be + scheduled, the cookies defined in the original + :class:`~scrapy.http.Request` object are no longer copied into the new + :class:`~scrapy.http.Request` object. + + If you manually set the ``Cookie`` header on a + :class:`~scrapy.http.Request` object and the domain name of the redirect + URL is not an exact match for the domain of the URL of the original + :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.http.Request` object. + + The old behavior could be exploited by an attacker to gain access to your + cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more + information. + + .. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8 + + .. note:: It is still possible to enable the sharing of cookies between + different domains with a shared domain suffix (e.g. + ``example.com`` and any subdomain) by defining the shared domain + suffix (e.g. ``example.com``) as the cookie domain when defining + your cookies. See the documentation of the + :class:`~scrapy.http.Request` class for more information. + +- When the domain of a cookie, either received in the ``Set-Cookie`` header + of a response or defined in a :class:`~scrapy.http.Request` object, is set + to a `public suffix `_, the cookie is now + ignored unless the cookie domain is the same as the request domain. + + The old behavior could be exploited by an attacker to inject cookies from a + controlled domain into your cookiejar that could be sent to other domains + not controlled by the attacker. Please, see the `mfjm-vh54-3f96 security + advisory`_ for more information. + + .. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96 + + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The h2_ dependency is now optional, only needed to + :ref:`enable HTTP/2 support `. (:issue:`5113`) + + .. _h2: https://pypi.org/project/h2/ + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The ``formdata`` parameter of :class:`~scrapy.FormRequest`, if specified + for a non-POST request, now overrides the URL query string, instead of + being appended to it. (:issue:`2919`, :issue:`3579`) + +- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, now + the return value of that function, and not the ``params`` input parameter, + will determine the feed URI parameters, unless that return value is + ``None``. (:issue:`4962`, :issue:`4966`) + +- In :class:`scrapy.core.engine.ExecutionEngine`, methods + :meth:`~scrapy.core.engine.ExecutionEngine.crawl`, + :meth:`~scrapy.core.engine.ExecutionEngine.download`, + :meth:`~scrapy.core.engine.ExecutionEngine.schedule`, + and :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle` + now raise :exc:`RuntimeError` if called before + :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`. (:issue:`5090`) + + These methods used to assume that + :attr:`ExecutionEngine.slot ` had + been defined by a prior call to + :meth:`~scrapy.core.engine.ExecutionEngine.open_spider`, so they were + raising :exc:`AttributeError` instead. + +- If the API of the configured :ref:`scheduler ` does not + meet expectations, :exc:`TypeError` is now raised at startup time. Before, + other exceptions would be raised at run time. (:issue:`3559`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- ``scrapy.http.TextResponse.body_as_unicode``, deprecated in Scrapy 2.2, has + now been removed. (:issue:`5393`) + +- ``scrapy.item.BaseItem``, deprecated in Scrapy 2.2, has now been removed. + (:issue:`5398`) + +- ``scrapy.item.DictItem``, deprecated in Scrapy 1.8, has now been removed. + (:issue:`5398`) + +- ``scrapy.Spider.make_requests_from_url``, deprecated in Scrapy 1.4, has now + been removed. (:issue:`4178`, :issue:`4356`) + + +Deprecations +~~~~~~~~~~~~ + +- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, + returning ``None`` or modifying the ``params`` input parameter is now + deprecated. Return a new dictionary instead. (:issue:`4962`, :issue:`4966`) + +- :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`) + + - Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new + :meth:`Request.to_dict ` method. + + - Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new + :func:`scrapy.utils.request.request_from_dict` function. + +- In :mod:`scrapy.squeues`, the following queue classes are deprecated: + :class:`~scrapy.squeues.PickleFifoDiskQueueNonRequest`, + :class:`~scrapy.squeues.PickleLifoDiskQueueNonRequest`, + :class:`~scrapy.squeues.MarshalFifoDiskQueueNonRequest`, + and :class:`~scrapy.squeues.MarshalLifoDiskQueueNonRequest`. You should + instead use: + :class:`~scrapy.squeues.PickleFifoDiskQueue`, + :class:`~scrapy.squeues.PickleLifoDiskQueue`, + :class:`~scrapy.squeues.MarshalFifoDiskQueue`, + and :class:`~scrapy.squeues.MarshalLifoDiskQueue`. (:issue:`5117`) + +- Many aspects of :class:`scrapy.core.engine.ExecutionEngine` that come from + a time when this class could handle multiple :class:`~scrapy.Spider` + objects at a time have been deprecated. (:issue:`5090`) + + - The :meth:`~scrapy.core.engine.ExecutionEngine.has_capacity` method + is deprecated. + + - The :meth:`~scrapy.core.engine.ExecutionEngine.schedule` method is + deprecated, use :meth:`~scrapy.core.engine.ExecutionEngine.crawl` or + :meth:`~scrapy.core.engine.ExecutionEngine.download` instead. + + - The :attr:`~scrapy.core.engine.ExecutionEngine.open_spiders` attribute + is deprecated, use :attr:`~scrapy.core.engine.ExecutionEngine.spider` + instead. + + - The ``spider`` parameter is deprecated for the following methods: + + - :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle` + + - :meth:`~scrapy.core.engine.ExecutionEngine.crawl` + + - :meth:`~scrapy.core.engine.ExecutionEngine.download` + + Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider` + first to set the :class:`~scrapy.Spider` object. + + +New features +~~~~~~~~~~~~ + +- You can now use :ref:`item filtering ` to control which items + are exported to each output feed. (:issue:`4575`, :issue:`5178`, + :issue:`5161`, :issue:`5203`) + +- You can now apply :ref:`post-processing ` to feeds, and + :ref:`built-in post-processing plugins ` are provided for + output file compression. (:issue:`2174`, :issue:`5168`, :issue:`5190`) + +- The :setting:`FEEDS` setting now supports :class:`pathlib.Path` objects as + keys. (:issue:`5383`, :issue:`5384`) + +- Enabling :ref:`asyncio ` while using Windows and Python 3.8 + or later will automatically switch the asyncio event loop to one that + allows Scrapy to work. See :ref:`asyncio-windows`. (:issue:`4976`, + :issue:`5315`) + +- The :command:`genspider` command now supports a start URL instead of a + domain name. (:issue:`4439`) + +- :mod:`scrapy.utils.defer` gained 2 new functions, + :func:`~scrapy.utils.defer.deferred_to_future` and + :func:`~scrapy.utils.defer.maybe_deferred_to_future`, to help :ref:`await + on Deferreds when using the asyncio reactor `. + (:issue:`5288`) + +- :ref:`Amazon S3 feed export storage ` gained + support for `temporary security credentials`_ + (:setting:`AWS_SESSION_TOKEN`) and endpoint customization + (:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`) + + .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys + +- New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file. + (:issue:`5279`) + +- :attr:`Request.cookies ` values that are + :class:`bool`, :class:`float` or :class:`int` are cast to :class:`str`. + (:issue:`5252`, :issue:`5253`) + +- You may now raise :exc:`~scrapy.exceptions.CloseSpider` from a handler of + the :signal:`spider_idle` signal to customize the reason why the spider is + stopping. (:issue:`5191`) + +- When using + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`, the + proxy URL for non-HTTPS HTTP/1.1 requests no longer needs to include a URL + scheme. (:issue:`4505`, :issue:`4649`) + +- All built-in queues now expose a ``peek`` method that returns the next + queue object (like ``pop``) but does not remove the returned object from + the queue. (:issue:`5112`) + + If the underlying queue does not support peeking (e.g. because you are not + using ``queuelib`` 1.6.1 or later), the ``peek`` method raises + :exc:`NotImplementedError`. + +- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have + an ``attributes`` attribute that makes subclassing easier. For + :class:`~scrapy.http.Request`, it also allows subclasses to work with + :func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`, + :issue:`5130`, :issue:`5218`) + +- The :meth:`~scrapy.core.scheduler.BaseScheduler.open` and + :meth:`~scrapy.core.scheduler.BaseScheduler.close` methods of the + :ref:`scheduler ` are now optional. (:issue:`3559`) + +- HTTP/1.1 :exc:`~scrapy.core.downloader.handlers.http11.TunnelError` + exceptions now only truncate response bodies longer than 1000 characters, + instead of those longer than 32 characters, making it easier to debug such + errors. (:issue:`4881`, :issue:`5007`) + +- :class:`~scrapy.loader.ItemLoader` now supports non-text responses. + (:issue:`5145`, :issue:`5269`) + + +Bug fixes +~~~~~~~~~ + +- The :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` settings + are no longer ignored if defined in :attr:`~scrapy.Spider.custom_settings`. + (:issue:`4485`, :issue:`5352`) + +- Removed a module-level Twisted reactor import that could prevent + :ref:`using the asyncio reactor `. (:issue:`5357`) + +- The :command:`startproject` command works with existing folders again. + (:issue:`4665`, :issue:`4676`) + +- The :setting:`FEED_URI_PARAMS` setting now behaves as documented. + (:issue:`4962`, :issue:`4966`) + +- :attr:`Request.cb_kwargs ` once again allows the + ``callback`` keyword. (:issue:`5237`, :issue:`5251`, :issue:`5264`) + +- Made :func:`scrapy.utils.response.open_in_browser` support more complex + HTML. (:issue:`5319`, :issue:`5320`) + +- Fixed :attr:`CSVFeedSpider.quotechar + ` being interpreted as the CSV file + encoding. (:issue:`5391`, :issue:`5394`) + +- Added missing setuptools_ to the list of dependencies. (:issue:`5122`) + + .. _setuptools: https://pypi.org/project/setuptools/ + +- :class:`LinkExtractor ` + now also works as expected with links that have comma-separated ``rel`` + attribute values including ``nofollow``. (:issue:`5225`) + +- Fixed a :exc:`TypeError` that could be raised during :ref:`feed export + ` parameter parsing. (:issue:`5359`) + + +Documentation +~~~~~~~~~~~~~ + +- :ref:`asyncio support ` is no longer considered + experimental. (:issue:`5332`) + +- Included :ref:`Windows-specific help for asyncio usage `. + (:issue:`4976`, :issue:`5315`) + +- Rewrote :ref:`topics-headless-browsing` with up-to-date best practices. + (:issue:`4484`, :issue:`4613`) + +- Documented :ref:`local file naming in media pipelines + `. (:issue:`5069`, :issue:`5152`) + +- :ref:`faq` now covers spider file name collision issues. (:issue:`2680`, + :issue:`3669`) + +- Provided better context and instructions to disable the + :setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`) + +- Documented that :ref:`reppy-parser` does not support Python 3.9+. + (:issue:`5226`, :issue:`5231`) + +- Documented :ref:`the scheduler component `. + (:issue:`3537`, :issue:`3559`) + +- Documented the method used by :ref:`media pipelines + ` to :ref:`determine if a file has expired + `. (:issue:`5120`, :issue:`5254`) + +- :ref:`run-multiple-spiders` now features + :func:`scrapy.utils.project.get_project_settings` usage. (:issue:`5070`) + +- :ref:`run-multiple-spiders` now covers what happens when you define + different per-spider values for some settings that cannot differ at run + time. (:issue:`4485`, :issue:`5352`) + +- Extended the documentation of the + :class:`~scrapy.extensions.statsmailer.StatsMailer` extension. + (:issue:`5199`, :issue:`5217`) + +- Added :setting:`JOBDIR` to :ref:`topics-settings`. (:issue:`5173`, + :issue:`5224`) + +- Documented :attr:`Spider.attribute `. + (:issue:`5174`, :issue:`5244`) + +- Documented :attr:`TextResponse.urljoin `. + (:issue:`1582`) + +- Added the ``body_length`` parameter to the documented signature of the + :signal:`headers_received` signal. (:issue:`5270`) + +- Clarified :meth:`SelectorList.get ` usage + in the :ref:`tutorial `. (:issue:`5256`) + +- The documentation now features the shortest import path of classes with + multiple import paths. (:issue:`2733`, :issue:`5099`) + +- ``quotes.toscrape.com`` references now use HTTPS instead of HTTP. + (:issue:`5395`, :issue:`5396`) + +- Added a link to `our Discord server `_ + to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`) + +- The pronunciation of the project name is now :ref:`officially + ` /ˈskreɪpaɪ/. (:issue:`5280`, :issue:`5281`) + +- Added the Scrapy logo to the README. (:issue:`5255`, :issue:`5258`) + +- Fixed issues and implemented minor improvements. (:issue:`3155`, + :issue:`4335`, :issue:`5074`, :issue:`5098`, :issue:`5134`, :issue:`5180`, + :issue:`5194`, :issue:`5239`, :issue:`5266`, :issue:`5271`, :issue:`5273`, + :issue:`5274`, :issue:`5276`, :issue:`5347`, :issue:`5356`, :issue:`5414`, + :issue:`5415`, :issue:`5416`, :issue:`5419`, :issue:`5420`) + + +Quality Assurance +~~~~~~~~~~~~~~~~~ + +- Added support for Python 3.10. (:issue:`5212`, :issue:`5221`, + :issue:`5265`) + +- Significantly reduced memory usage by + :func:`scrapy.utils.response.response_httprepr`, used by the + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` downloader + middleware, which is enabled by default. (:issue:`4964`, :issue:`4972`) + +- Removed uses of the deprecated :mod:`optparse` module. (:issue:`5366`, + :issue:`5374`) + +- Extended typing hints. (:issue:`5077`, :issue:`5090`, :issue:`5100`, + :issue:`5108`, :issue:`5171`, :issue:`5215`, :issue:`5334`) + +- Improved tests, fixed CI issues, removed unused code. (:issue:`5094`, + :issue:`5157`, :issue:`5162`, :issue:`5198`, :issue:`5207`, :issue:`5208`, + :issue:`5229`, :issue:`5298`, :issue:`5299`, :issue:`5310`, :issue:`5316`, + :issue:`5333`, :issue:`5388`, :issue:`5389`, :issue:`5400`, :issue:`5401`, + :issue:`5404`, :issue:`5405`, :issue:`5407`, :issue:`5410`, :issue:`5412`, + :issue:`5425`, :issue:`5427`) + +- Implemented improvements for contributors. (:issue:`5080`, :issue:`5082`, + :issue:`5177`, :issue:`5200`) + +- Implemented cleanups. (:issue:`5095`, :issue:`5106`, :issue:`5209`, + :issue:`5228`, :issue:`5235`, :issue:`5245`, :issue:`5246`, :issue:`5292`, + :issue:`5314`, :issue:`5322`) + + +.. _release-2.5.1: + +Scrapy 2.5.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + +.. _release-2.5.0: + +Scrapy 2.5.0 (2021-04-06) +------------------------- + +Highlights: + +- Official Python 3.9 support + +- Experimental :ref:`HTTP/2 support ` + +- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function + to retry requests from spider callbacks + +- New :class:`~scrapy.signals.headers_received` signal that allows stopping + downloads early + +- New :class:`Response.protocol ` attribute + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed all code that :ref:`was deprecated in 1.7.0 <1.7-deprecations>` and + had not :ref:`already been removed in 2.4.0 <2.4-deprecation-removals>`. + (:issue:`4901`) + +- Removed support for the ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment + variable, :ref:`deprecated in 1.8.0 <1.8-deprecations>`. (:issue:`4912`) + + +Deprecations +~~~~~~~~~~~~ + +- The :mod:`scrapy.utils.py36` module is now deprecated in favor of + :mod:`scrapy.utils.asyncgen`. (:issue:`4900`) + + +New features +~~~~~~~~~~~~ + +- Experimental :ref:`HTTP/2 support ` through a new download handler + that can be assigned to the ``https`` protocol in the + :setting:`DOWNLOAD_HANDLERS` setting. + (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) + +- The new :func:`scrapy.downloadermiddlewares.retry.get_retry_request` + function may be used from spider callbacks or middlewares to handle the + retrying of a request beyond the scenarios that + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` supports. + (:issue:`3590`, :issue:`3685`, :issue:`4902`) + +- The new :class:`~scrapy.signals.headers_received` signal gives early access + to response headers and allows :ref:`stopping downloads + `. + (:issue:`1772`, :issue:`4897`) + +- The new :attr:`Response.protocol ` + attribute gives access to the string that identifies the protocol used to + download a response. (:issue:`4878`) + +- :ref:`Stats ` now include the following entries that indicate + the number of successes and failures in storing + :ref:`feeds `:: + + feedexport/success_count/ + feedexport/failed_count/ + + Where ```` is the feed storage backend class name, such as + :class:`~scrapy.extensions.feedexport.FileFeedStorage` or + :class:`~scrapy.extensions.feedexport.FTPFeedStorage`. + + (:issue:`3947`, :issue:`4850`) + +- The :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` spider + middleware now logs ignored URLs with ``INFO`` :ref:`logging level + ` instead of ``DEBUG``, and it now includes the following entry + into :ref:`stats ` to keep track of the number of ignored + URLs:: + + urllength/request_ignored_count + + (:issue:`5036`) + +- The + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + downloader middleware now logs the number of decompressed responses and the + total count of resulting bytes:: + + httpcompression/response_bytes + httpcompression/response_count + + (:issue:`4797`, :issue:`4799`) + + +Bug fixes +~~~~~~~~~ + +- Fixed installation on PyPy installing PyDispatcher in addition to + PyPyDispatcher, which could prevent Scrapy from working depending on which + package got imported. (:issue:`4710`, :issue:`4814`) + +- When inspecting a callback to check if it is a generator that also returns + a value, an exception is no longer raised if the callback has a docstring + with lower indentation than the following code. + (:issue:`4477`, :issue:`4935`) + +- The `Content-Length `_ + header is no longer omitted from responses when using the default, HTTP/1.1 + download handler (see :setting:`DOWNLOAD_HANDLERS`). + (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`) + +- Setting the :reqmeta:`handle_httpstatus_all` request meta key to ``False`` + now has the same effect as not setting it at all, instead of having the + same effect as setting it to ``True``. + (:issue:`3851`, :issue:`4694`) + + +Documentation +~~~~~~~~~~~~~ + +- Added instructions to :ref:`install Scrapy in Windows using pip + `. + (:issue:`4715`, :issue:`4736`) + +- Logging documentation now includes :ref:`additional ways to filter logs + `. + (:issue:`4216`, :issue:`4257`, :issue:`4965`) + +- Covered how to deal with long lists of allowed domains in the :ref:`FAQ + `. (:issue:`2263`, :issue:`3667`) + +- Covered scrapy-bench_ in :ref:`benchmarking`. + (:issue:`4996`, :issue:`5016`) + +- Clarified that one :ref:`extension ` instance is created + per crawler. + (:issue:`5014`) + +- Fixed some errors in examples. + (:issue:`4829`, :issue:`4830`, :issue:`4907`, :issue:`4909`, + :issue:`5008`) + +- Fixed some external links, typos, and so on. + (:issue:`4892`, :issue:`4899`, :issue:`4936`, :issue:`4942`, :issue:`5005`, + :issue:`5063`) + +- The :ref:`list of Request.meta keys ` is now sorted + alphabetically. + (:issue:`5061`, :issue:`5065`) + +- Updated references to Scrapinghub, which is now called Zyte. + (:issue:`4973`, :issue:`5072`) + +- Added a mention to contributors in the README. (:issue:`4956`) + +- Reduced the top margin of lists. (:issue:`4974`) + + +Quality Assurance +~~~~~~~~~~~~~~~~~ + +- Made Python 3.9 support official (:issue:`4757`, :issue:`4759`) + +- Extended typing hints (:issue:`4895`) + +- Fixed deprecated uses of the Twisted API. + (:issue:`4940`, :issue:`4950`, :issue:`5073`) + +- Made our tests run with the new pip resolver. + (:issue:`4710`, :issue:`4814`) + +- Added tests to ensure that :ref:`coroutine support ` + is tested. (:issue:`4987`) + +- Migrated from Travis CI to GitHub Actions. (:issue:`4924`) + +- Fixed CI issues. + (:issue:`4986`, :issue:`5020`, :issue:`5022`, :issue:`5027`, :issue:`5052`, + :issue:`5053`) + +- Implemented code refactorings, style fixes and cleanups. + (:issue:`4911`, :issue:`4982`, :issue:`5001`, :issue:`5002`, :issue:`5076`) + + .. _release-2.4.1: Scrapy 2.4.1 (2020-11-17) @@ -97,6 +725,8 @@ Backward-incompatible changes (:issue:`4717`, :issue:`4823`) +.. _2.4-deprecation-removals: + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ @@ -754,9 +1384,8 @@ Bug fixes * zope.interface 5.0.0 and later versions are now supported (:issue:`4447`, :issue:`4448`) -* :meth:`Spider.make_requests_from_url - `, deprecated in Scrapy - 1.4.0, now issues a warning when used (:issue:`4412`) +* ``Spider.make_requests_from_url``, deprecated in Scrapy 1.4.0, now issues a + warning when used (:issue:`4412`) Documentation @@ -1014,7 +1643,7 @@ New features :issue:`4370`) * A new ``keep_fragments`` parameter of - :func:`scrapy.utils.request.request_fingerprint` allows to generate + ``scrapy.utils.request.request_fingerprint`` allows to generate different fingerprints for requests with different fragments in their URL (:issue:`4104`) @@ -1268,6 +1897,88 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.2: + +Scrapy 1.8.2 (2022-03-01) +------------------------- + +**Security bug fixes:** + +- When a :class:`~scrapy.http.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.http.Request` object to be + scheduled, the cookies defined in the original + :class:`~scrapy.http.Request` object are no longer copied into the new + :class:`~scrapy.http.Request` object. + + If you manually set the ``Cookie`` header on a + :class:`~scrapy.http.Request` object and the domain name of the redirect + URL is not an exact match for the domain of the URL of the original + :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.http.Request` object. + + The old behavior could be exploited by an attacker to gain access to your + cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more + information. + + .. _cjvr-mfj7-j4j8 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cjvr-mfj7-j4j8 + + .. note:: It is still possible to enable the sharing of cookies between + different domains with a shared domain suffix (e.g. + ``example.com`` and any subdomain) by defining the shared domain + suffix (e.g. ``example.com``) as the cookie domain when defining + your cookies. See the documentation of the + :class:`~scrapy.http.Request` class for more information. + +- When the domain of a cookie, either received in the ``Set-Cookie`` header + of a response or defined in a :class:`~scrapy.http.Request` object, is set + to a `public suffix `_, the cookie is now + ignored unless the cookie domain is the same as the request domain. + + The old behavior could be exploited by an attacker to inject cookies into + your requests to some other domains. Please, see the `mfjm-vh54-3f96 + security advisory`_ for more information. + + .. _mfjm-vh54-3f96 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-mfjm-vh54-3f96 + + +.. _release-1.8.1: + +Scrapy 1.8.1 (2021-10-05) +------------------------- + +* **Security bug fix:** + + If you use + :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` + (i.e. the ``http_user`` and ``http_pass`` spider attributes) for HTTP + authentication, any request exposes your credentials to the request target. + + To prevent unintended exposure of authentication credentials to unintended + domains, you must now additionally set a new, additional spider attribute, + ``http_auth_domain``, and point it to the specific domain to which the + authentication credentials must be sent. + + If the ``http_auth_domain`` spider attribute is not set, the domain of the + first request will be considered the HTTP authentication target, and + authentication credentials will only be sent in requests targeting that + domain. + + If you need to send the same HTTP authentication credentials to multiple + domains, you can use :func:`w3lib.http.basic_auth_header` instead to + set the value of the ``Authorization`` header of your requests. + + If you *really* want your spider to send the same HTTP authentication + credentials to any domain, set the ``http_auth_domain`` spider attribute + to ``None``. + + Finally, if you are a user of `scrapy-splash`_, know that this version of + Scrapy breaks compatibility with scrapy-splash 0.7.2 and earlier. You will + need to upgrade scrapy-splash to a greater version for it to continue to + work. + +.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash + + .. _release-1.8.0: Scrapy 1.8.0 (2019-10-28) @@ -1433,6 +2144,8 @@ Deprecation removals * ``scrapy.xlib`` has been removed (:issue:`4015`) +.. _1.8-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -1566,7 +2279,7 @@ New features * A new scheduler priority queue, ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant - scheduling improvement on crawls targetting multiple web domains, at the + scheduling improvement on crawls targeting multiple web domains, at the cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) * A new :attr:`Request.cb_kwargs ` attribute @@ -1789,6 +2502,8 @@ The following deprecated settings have also been removed (:issue:`3578`): * ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`) +.. _1.7-deprecations: + Deprecations ~~~~~~~~~~~~ @@ -2428,7 +3143,7 @@ Bug fixes - Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`). - Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`). - Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``, - ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). + ``DOWNLOADER_MIDDLEWARES``, ``EXTENSIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). Documentation ~~~~~~~~~~~~~ @@ -2602,7 +3317,7 @@ Bug fixes - Fix for selected callbacks when using ``CrawlSpider`` with :command:`scrapy parse ` (:issue:`2225`). - Fix for invalid JSON and XML files when spider yields no items (:issue:`872`). -- Implement ``flush()`` fpr ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). +- Implement ``flush()`` for ``StreamLogger`` avoiding a warning in logs (:issue:`2125`). Refactoring ~~~~~~~~~~~ @@ -3465,7 +4180,7 @@ Scrapy 0.24.3 (2014-08-09) - adding some xpath tips to selectors docs (:commit:`2d103e0`) - fix tests to account for https://github.com/scrapy/w3lib/pull/23 (:commit:`f8d366a`) - get_func_args maximum recursion fix #728 (:commit:`81344ea`) -- Updated input/ouput processor example according to #560. (:commit:`f7c4ea8`) +- Updated input/output processor example according to #560. (:commit:`f7c4ea8`) - Fixed Python syntax in tutorial. (:commit:`db59ed9`) - Add test case for tunneling proxy (:commit:`f090260`) - Bugfix for leaking Proxy-Authorization header to remote host when using tunneling (:commit:`d8793af`) @@ -4092,7 +4807,7 @@ Code rearranged and removed - Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot - Removed support for default field values in Scrapy items (:rev:`2616`) - Removed experimental crawlspider v2 (:rev:`2632`) -- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`) +- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`) - Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`) - Removed deprecated Execution Queue (:rev:`2704`) - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) @@ -4127,7 +4842,7 @@ Scrapyd changes ~~~~~~~~~~~~~~~ - Scrapyd now uses one process per spider -- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default) +- It stores one log file per spider run, and rotate them keeping the latest 5 logs per spider (by default) - A minimal web ui was added, available at http://localhost:6800 by default - There is now a ``scrapy server`` command to start a Scrapyd server of the current project @@ -4163,7 +4878,7 @@ New features and improvements - Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195) - Support for overriding default request headers per spider (#181) - Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186) -- Splitted Debian package into two packages - the library and the service (#187) +- Split Debian package into two packages - the library and the service (#187) - Scrapy log refactoring (#188) - New extension for keeping persistent spider contexts among different runs (#203) - Added ``dont_redirect`` request.meta key for avoiding redirects (#233) @@ -4184,7 +4899,7 @@ API changes - ``url`` and ``body`` attributes of Request objects are now read-only (#230) - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) -- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) +- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) - Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself. - Changes to Scrapy Manager (now called "Crawler"): - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` @@ -4331,6 +5046,7 @@ First release of Scrapy. .. _resource: https://docs.python.org/2/library/resource.html .. _robots.txt: https://www.robotstxt.org/ .. _scrapely: https://github.com/scrapy/scrapely +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ diff --git a/docs/requirements.txt b/docs/requirements.txt index 3d34b47da..a0930ba1e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ Sphinx>=3.0 sphinx-hoverxref>=0.2b1 sphinx-notfound-page>=0.4 -sphinx_rtd_theme>=0.4 +sphinx-rtd-theme>=0.5.2 \ No newline at end of file diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 445b2979f..60b5acd10 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -29,9 +29,16 @@ how you :ref:`configure the downloader middlewares .. class:: Crawler(spidercls, settings) The Crawler object must be instantiated with a - :class:`scrapy.spiders.Spider` subclass and a + :class:`scrapy.Spider` subclass and a :class:`scrapy.settings.Settings` object. + .. attribute:: request_fingerprinter + + The request fingerprint builder of this crawler. + + This is used from extensions and middlewares to build short, unique + identifiers for requests. See :ref:`request-fingerprints`. + .. attribute:: settings The settings manager of this crawler. @@ -196,7 +203,7 @@ SpiderLoader API match the request's url against the domains of the spiders. :param request: queried request - :type request: :class:`~scrapy.http.Request` instance + :type request: :class:`~scrapy.Request` instance .. _topics-api-signals: diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 074c59241..0c3a7ed88 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -67,7 +67,7 @@ this: the :ref:`Scheduler ` and asks for possible next Requests to crawl. -9. The process repeats (from step 1) until there are no more requests from the +9. The process repeats (from step 3) until there are no more requests from the :ref:`Scheduler `. Components @@ -87,8 +87,9 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The Scheduler receives requests from the engine and enqueues them for feeding -them later (also to the engine) when the engine requests them. +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine +requests them. .. _component-downloader: diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 91e1cca0d..3a6941a2c 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -6,13 +6,10 @@ asyncio .. versionadded:: 2.0 -Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio -reactor `, you may use :mod:`asyncio` and +Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the +asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy - versions may introduce related changes without a deprecation - period or warning. .. _install-asyncio: @@ -29,6 +26,7 @@ reactor manually. You can do that using install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + .. _using-custom-loops: Using custom asyncio loops @@ -39,4 +37,62 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the use it instead of the default asyncio event loop. +.. _asyncio-windows: +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations: + +- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required + when using Twisted. + +- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work + with Twisted. + +So on Python 3.8+ the event loop class needs to be changed. + +.. versionchanged:: 2.6.0 + The event loop class is changed automatically when you change the + :setting:`TWISTED_REACTOR` setting or call + :func:`~scrapy.utils.reactor.install_reactor`. + +To change the event loop class manually, call the following code before +installing the reactor:: + + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +You can put this in the same function that installs the reactor, if you do that +yourself, or in some code that runs before the reactor is installed, e.g. +``settings.py``. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + + +.. _asyncio-await-dfd: + +Awaiting on Deferreds +===================== + +When the asyncio reactor isn't installed, you can await on Deferreds in the +coroutines directly. When it is installed, this is not possible anymore, due to +specifics of the Scrapy coroutine integration (the coroutines are wrapped into +:class:`asyncio.Future` objects, not into +:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into +Futures. Scrapy provides two helpers for this: + +.. autofunction:: scrapy.utils.defer.deferred_to_future +.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future +.. tip:: If you need to use these functions in code that aims to be compatible + with lower versions of Scrapy that do not provide these functions, + down to Scrapy 2.0 (earlier versions do not support + :mod:`asyncio`), you can copy the implementation of these functions + into your own code. diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index b01a66188..0643df6a6 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -81,5 +81,6 @@ follow links, any custom spider you write will probably do more stuff which results in slower crawl rates. How slower depends on how much your spider does and how well it's written. -In the future, more cases will be added to the benchmarking suite to cover -other common scenarios. +Use scrapy-bench_ for more complex benchmarking. + +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench \ No newline at end of file diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 7de5e8121..8c0b8e55f 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -230,10 +230,16 @@ Usage example:: genspider --------- -* Syntax: ``scrapy genspider [-t template] `` +* Syntax: ``scrapy genspider [-t template] `` * Requires project: *no* -Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. +.. versionadded:: 2.6.0 + The ability to pass a URL instead of a domain. + +Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. + +.. note:: Even if an HTTPS URL is specified, the protocol used in + ``start_urls`` is always HTTP. This is a known issue: :issue:`3553`. Usage example:: @@ -598,8 +604,6 @@ Example: Register commands via setup.py entry points ------------------------------------------- -.. note:: This is an experimental feature, use with caution. - You can also add Scrapy commands from an external library by adding a ``scrapy.commands`` section in the entry points of the library ``setup.py`` file. diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index e61421bf1..ef296dc9e 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -37,7 +37,7 @@ This callback is tested using three built-in contracts: .. class:: CallbackKeywordArgumentsContract - This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` + This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` attribute for the sample request. It must be a valid JSON dictionary. :: @@ -88,7 +88,7 @@ override three methods: .. method:: Contract.adjust_request_args(args) This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.http.Request` is used by default, + for request object. :class:`~scrapy.Request` is used by default, but this can be changed with the ``request_cls`` attribute. If multiple contracts in chain have this attribute defined, the last one is used. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3b1549bd3..549552bd1 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -1,3 +1,5 @@ +.. _topics-coroutines: + ========== Coroutines ========== @@ -15,7 +17,7 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): -- :class:`~scrapy.http.Request` callbacks. +- :class:`~scrapy.Request` callbacks. .. note:: The callback output is not processed until the whole callback finishes. @@ -75,23 +77,28 @@ coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. This means you can use many useful Python libraries providing such code:: - class MySpider(Spider): + class MySpiderDeferred(Spider): # ... - async def parse_with_deferred(self, response): + async def parse(self, response): additional_response = await treq.get('https://additional.url') additional_data = await treq.content(additional_response) # ... use response and additional_data to yield items and requests - async def parse_with_asyncio(self, response): + class MySpiderAsyncio(Spider): + # ... + async def parse(self, response): async with aiohttp.ClientSession() as session: async with session.get('https://additional.url') as additional_response: - additional_data = await r.text() + additional_data = await additional_response.text() # ... use response and additional_data to yield items and requests .. note:: Many libraries that use coroutines, such as `aio-libs`_, require the :mod:`asyncio` loop and to use them you need to :doc:`enable asyncio support in Scrapy`. +.. note:: If you want to ``await`` on Deferreds while using the asyncio reactor, + you need to :ref:`wrap them`. + Common use cases for asynchronous code include: * requesting data from websites, databases and other services (in callbacks, diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index d75f17301..4d452b4df 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -36,7 +36,7 @@ Consider the following Scrapy spider below:: Basically this is a simple spider which parses two pages of items (the start_urls). Items also have a details page with additional information, so we -use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a +use the ``cb_kwargs`` functionality of :class:`~scrapy.Request` to pass a partially populated item. diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index 361914a29..961d6dc01 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -14,7 +14,7 @@ spiders come in. Popular choices for deploying Scrapy spiders are: * :ref:`Scrapyd ` (open source) -* :ref:`Scrapy Cloud ` (cloud-based) +* :ref:`Zyte Scrapy Cloud ` (cloud-based) .. _deploy-scrapyd: @@ -32,28 +32,28 @@ Scrapyd is maintained by some of the Scrapy developers. .. _deploy-scrapy-cloud: -Deploying to Scrapy Cloud -========================= +Deploying to Zyte Scrapy Cloud +============================== -`Scrapy Cloud`_ is a hosted, cloud-based service by `Scrapinghub`_, -the company behind Scrapy. +`Zyte Scrapy Cloud`_ is a hosted, cloud-based service by Zyte_, the company +behind Scrapy. -Scrapy Cloud removes the need to setup and monitor servers -and provides a nice UI to manage spiders and review scraped items, -logs and stats. +Zyte Scrapy Cloud removes the need to setup and monitor servers and provides a +nice UI to manage spiders and review scraped items, logs and stats. -To deploy spiders to Scrapy Cloud you can use the `shub`_ command line tool. -Please refer to the `Scrapy Cloud documentation`_ for more information. +To deploy spiders to Zyte Scrapy Cloud you can use the `shub`_ command line +tool. +Please refer to the `Zyte Scrapy Cloud documentation`_ for more information. -Scrapy Cloud is compatible with Scrapyd and one can switch between +Zyte Scrapy Cloud is compatible with Scrapyd and one can switch between them as needed - the configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``. -.. _Scrapyd: https://github.com/scrapy/scrapyd .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud +.. _Scrapyd: https://github.com/scrapy/scrapyd .. _scrapyd-client: https://github.com/scrapy/scrapyd-client -.. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html -.. _Scrapinghub: https://scrapinghub.com/ +.. _shub: https://shub.readthedocs.io/en/latest/ +.. _Zyte: https://zyte.com/ +.. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/ +.. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index c83b1a9d9..9bf97c628 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -19,14 +19,14 @@ Caveats with inspecting the live browser DOM Since Developer Tools operate on a live browser DOM, what you'll actually see when inspecting the page source is not the original HTML, but a modified one -after applying some browser clean up and executing Javascript code. Firefox, +after applying some browser clean up and executing JavaScript code. Firefox, in particular, is known for adding ```` elements to tables. Scrapy, on the other hand, does not modify the original page HTML, so you won't be able to extract any data if you use ```` in your XPath expressions. Therefore, you should keep in mind the following things: -* Disable Javascript while inspecting the DOM looking for XPaths to be +* Disable JavaScript while inspecting the DOM looking for XPaths to be used in Scrapy (in the Developer Tools settings click `Disable JavaScript`) * Never use full XPath paths, use relative and clever ones based on attributes @@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class= "text"`` we will see the quote-text we clicked on. The `Inspector` lets you copy XPaths to selected elements. Let's try it out. -First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal: +First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal: .. code-block:: none - $ scrapy shell "http://quotes.toscrape.com/" + $ scrapy shell "https://quotes.toscrape.com/" Then, back to your web browser, right-click on the ``span`` tag, select ``Copy > XPath`` and paste it in the Scrapy shell like so: .. invisible-code-block: python - response = load_response('http://quotes.toscrape.com/', 'quotes.html') + response = load_response('https://quotes.toscrape.com/', 'quotes.html') >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] @@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the type ``json``. If we click on this request, we see that the request URL is -``http://quotes.toscrape.com/api/quotes?page=1`` and the response +``https://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click on the request and open ``Open in new tab`` to get a better overview. @@ -247,7 +247,7 @@ also request each page to get every quote on the site:: name = 'quote' allowed_domains = ['quotes.toscrape.com'] page = 1 - start_urls = ['http://quotes.toscrape.com/api/quotes?page=1'] + start_urls = ['https://quotes.toscrape.com/api/quotes?page=1'] def parse(self, response): data = json.loads(response.text) @@ -255,7 +255,7 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" + url = f"https://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) This spider starts at the first page of the quotes-API. With each @@ -274,13 +274,13 @@ In more complex websites, it could be difficult to easily reproduce the requests, as we could need to add ``headers`` or ``cookies`` to make it work. In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the -:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent +:meth:`~scrapy.Request.from_curl()` method to generate an equivalent request:: from scrapy import Request request = Request.from_curl( - "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" + "curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce" "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X" "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM" @@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools -.. _quotes.toscrape.com: http://quotes.toscrape.com -.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll -.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10 +.. _quotes.toscrape.com: https://quotes.toscrape.com +.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll +.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 6801adc9c..29e350651 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -76,7 +76,7 @@ object gives you access, for example, to the :ref:`settings `. middleware. :meth:`process_request` should either: return ``None``, return a - :class:`~scrapy.http.Response` object, return a :class:`~scrapy.http.Request` + :class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request` object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`. If it returns ``None``, Scrapy will continue processing this request, executing all @@ -88,8 +88,8 @@ object gives you access, for example, to the :ref:`settings `. or the appropriate download function; it'll return that response. The :meth:`process_response` methods of installed middleware is always called on every response. - If it returns a :class:`~scrapy.http.Request` object, Scrapy will stop calling - process_request methods and reschedule the returned request. Once the newly returned + If it returns a :class:`~scrapy.Request` object, Scrapy will stop calling + :meth:`process_request` methods and reschedule the returned request. Once the newly returned request is performed, the appropriate middleware chain will be called on the downloaded response. @@ -100,22 +100,22 @@ object gives you access, for example, to the :ref:`settings `. ignored and not logged (unlike other exceptions). :param request: the request being processed - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_response(request, response, spider) :meth:`process_response` should either: return a :class:`~scrapy.http.Response` - object, return a :class:`~scrapy.http.Request` object or + object, return a :class:`~scrapy.Request` object or raise a :exc:`~scrapy.exceptions.IgnoreRequest` exception. If it returns a :class:`~scrapy.http.Response` (it could be the same given response, or a brand-new one), that response will continue to be processed with the :meth:`process_response` of the next middleware in the chain. - If it returns a :class:`~scrapy.http.Request` object, the middleware chain is + If it returns a :class:`~scrapy.Request` object, the middleware chain is halted and the returned request is rescheduled to be downloaded in the future. This is the same behavior as if a request is returned from :meth:`process_request`. @@ -124,13 +124,13 @@ object gives you access, for example, to the :ref:`settings `. exception, it is ignored and not logged (unlike other exceptions). :param request: the request that originated the response - :type request: is a :class:`~scrapy.http.Request` object + :type request: is a :class:`~scrapy.Request` object :param response: the response being processed :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_exception(request, exception, spider) @@ -139,7 +139,7 @@ object gives you access, for example, to the :ref:`settings `. exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception) :meth:`process_exception` should return: either ``None``, - a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.http.Request` object. + a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_exception` methods of installed middleware, @@ -149,19 +149,19 @@ object gives you access, for example, to the :ref:`settings `. method chain of installed middleware is started, and Scrapy won't bother calling any other :meth:`process_exception` methods of middleware. - If it returns a :class:`~scrapy.http.Request` object, the returned request is + If it returns a :class:`~scrapy.Request` object, the returned request is rescheduled to be downloaded in the future. This stops the execution of :meth:`process_exception` methods of the middleware the same as returning a response would. :param request: the request that generated the exception - :type request: is a :class:`~scrapy.http.Request` object + :type request: is a :class:`~scrapy.Request` object :param exception: the raised exception :type exception: an ``Exception`` object :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) @@ -203,13 +203,13 @@ CookiesMiddleware browsers do. .. caution:: When non-UTF8 encoded byte sequences are passed to a - :class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log + :class:`~scrapy.Request`, the ``CookiesMiddleware`` will log a warning. Refer to :ref:`topics-logging-advanced-customization` to customize the logging behaviour. .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. The following settings can be used to configure the cookie middleware: @@ -258,7 +258,7 @@ web server and received cookies in :class:`~scrapy.http.Response` will **not** be merged with the existing cookies. For more detailed information see the ``cookies`` parameter in -:class:`~scrapy.http.Request`. +:class:`~scrapy.Request`. .. setting:: COOKIES_DEBUG @@ -323,8 +323,21 @@ HttpAuthMiddleware This middleware authenticates all requests generated from certain spiders using `Basic access authentication`_ (aka. HTTP auth). - To enable HTTP authentication from certain spiders, set the ``http_user`` - and ``http_pass`` attributes of those spiders. + To enable HTTP authentication for a spider, set the ``http_user`` and + ``http_pass`` spider attributes to the authentication data and the + ``http_auth_domain`` spider attribute to the domain which requires this + authentication (its subdomains will be also handled in the same way). + You can set ``http_auth_domain`` to ``None`` to enable the + authentication for all requests but you risk leaking your authentication + credentials to unrelated domains. + + .. warning:: + In previous Scrapy versions HttpAuthMiddleware sent the authentication + data with all requests, which is a security problem if the spider + makes requests to several different domains. Currently if the + ``http_auth_domain`` attribute is not set, the middleware will use the + domain of the first request, which will work for some spiders but not + for others. In the future the middleware will produce an error instead. Example:: @@ -334,6 +347,7 @@ HttpAuthMiddleware http_user = 'someuser' http_pass = 'somepass' + http_auth_domain = 'intranet.example.com' name = 'intranet.example.com' # .. rest of the spider code omitted ... @@ -352,7 +366,7 @@ HttpCacheMiddleware This middleware provides low-level cache to all HTTP requests and responses. It has to be combined with a cache storage backend as well as a cache policy. - Scrapy ships with three HTTP cache storage backends: + Scrapy ships with the following HTTP cache storage backends: * :ref:`httpcache-storage-fs` * :ref:`httpcache-storage-dbm` @@ -501,7 +515,7 @@ defines the methods described below. the :signal:`open_spider ` signal. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: close_spider(spider) @@ -509,27 +523,27 @@ defines the methods described below. the :signal:`close_spider ` signal. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: retrieve_response(spider, request) Return response if present in cache, or ``None`` otherwise. :param spider: the spider which generated the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param request: the request to find cached response for - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object .. method:: store_response(spider, request, response) Store the given response in the cache. :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param request: the corresponding request the spider generated - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param response: the response to store in the cache :type response: :class:`~scrapy.http.Response` object @@ -690,14 +704,15 @@ HttpCompressionMiddleware sent/received from web sites. This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is + `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt -.. _brotlipy: https://pypi.org/project/brotlipy/ +.. _brotli: https://pypi.org/project/Brotli/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt .. _zstandard: https://pypi.org/project/zstandard/ + HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -722,7 +737,7 @@ HttpProxyMiddleware .. class:: HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the - ``proxy`` meta value for :class:`~scrapy.http.Request` objects. + ``proxy`` meta value for :class:`~scrapy.Request` objects. Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: @@ -749,12 +764,12 @@ RedirectMiddleware .. reqmeta:: redirect_urls The urls which the request goes through (while being redirected) can be found -in the ``redirect_urls`` :attr:`Request.meta ` key. +in the ``redirect_urls`` :attr:`Request.meta ` key. .. reqmeta:: redirect_reasons The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the -``redirect_reasons`` :attr:`Request.meta ` key. For +``redirect_reasons`` :attr:`Request.meta ` key. For example: ``[301, 302, 307, 'meta refresh']``. The format of a reason depends on the middleware that handled the corresponding @@ -770,7 +785,7 @@ settings (see the settings documentation for more info): .. reqmeta:: dont_redirect -If :attr:`Request.meta ` has ``dont_redirect`` +If :attr:`Request.meta ` has ``dont_redirect`` key set to True, the request will be ignored by this middleware. If you want to handle some redirect status codes in your spider, you can @@ -783,7 +798,7 @@ responses (and pass them through to your spider) you can do this:: handle_httpstatus_list = [301, 302] The ``handle_httpstatus_list`` key of :attr:`Request.meta -` can also be used to specify which response codes to +` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` to ``True`` if you want to allow any response code for a request. @@ -889,9 +904,14 @@ settings (see the settings documentation for more info): .. reqmeta:: dont_retry -If :attr:`Request.meta ` has ``dont_retry`` key +If :attr:`Request.meta ` has ``dont_retry`` key set to True, the request will be ignored by this middleware. +To retry requests from a spider callback, you can use the +:func:`get_retry_request` function: + +.. autofunction:: get_retry_request + RetryMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -914,7 +934,7 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. Maximum number of retries can also be specified per-request using -:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. +:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. When initialized, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. @@ -932,6 +952,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_PRIORITY_ADJUST + +RETRY_PRIORITY_ADJUST +--------------------- + +Default: ``-1`` + +Adjust retry request priority relative to original request: + +- a positive priority adjust means higher priority. +- **a negative priority adjust (default) means lower priority.** + .. _topics-dlmw-robots: @@ -969,7 +1001,7 @@ RobotsTxtMiddleware .. reqmeta:: dont_obey_robotstxt -If :attr:`Request.meta ` has +If :attr:`Request.meta ` has ``dont_obey_robotstxt`` key set to True the request will be ignored by this middleware even if :setting:`ROBOTSTXT_OBEY` is enabled. @@ -988,7 +1020,7 @@ Parsers vary in several aspects: (shorter) rule Performance comparison of different parsers is available at `the following link -`_. +`_. .. _protego-parser: @@ -1053,9 +1085,13 @@ In order to use this parser: * Install `Reppy `_ by running ``pip install reppy`` + .. warning:: `Upstream issue #122 + `_ prevents reppy usage in Python 3.9+. + * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` + .. _rerp-parser: Robotexclusionrulesparser diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 495111b56..ea5d06210 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -62,9 +62,9 @@ download the webpage with an HTTP client like curl_ or wget_ and see if the information can be found in the response they get. If they get a response with the desired data, modify your Scrapy -:class:`~scrapy.http.Request` to match that of the other HTTP client. For +:class:`~scrapy.Request` to match that of the other HTTP client. For example, try using the same user-agent string (:setting:`USER_AGENT`) or the -same :attr:`~scrapy.http.Request.headers`. +same :attr:`~scrapy.Request.headers`. If they also get a response without the desired data, you’ll need to take steps to make your request more similar to that of the web browser. See @@ -81,14 +81,14 @@ Use the :ref:`network tool ` of your web browser to see how your web browser performs the desired request, and try to reproduce that request with Scrapy. -It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP +It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.http.FormRequest`) of that request. +form parameters (see :class:`~scrapy.FormRequest`) of that request. As all major browsers allow to export the requests in `cURL `_ format, Scrapy incorporates the method -:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent -:class:`~scrapy.http.Request` from a cURL command. To get more information +:meth:`~scrapy.Request.from_curl()` to generate an equivalent +:class:`~scrapy.Request` from a cURL command. To get more information visit :ref:`request from curl ` inside the network tool section. @@ -125,7 +125,7 @@ data from it depends on the type of response: If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a - :class:`~scrapy.selector.Selector` and then + :class:`~scrapy.Selector` and then :ref:`use it ` as usual:: selector = Selector(data['html']) @@ -246,24 +246,46 @@ Using a headless browser ======================== A `headless browser`_ is a special web browser that provides an API for -automation. +automation. By installing the :ref:`asyncio reactor `, +it is possible to integrate ``asyncio``-based libraries which handle headless browsers. -The easiest way to use a headless browser with Scrapy is to use Selenium_, -along with `scrapy-selenium`_ for seamless integration. +One such library is `playwright-python`_ (an official Python port of `playwright`_). +The following is a simple snippet to illustrate its usage within a Scrapy spider:: + import scrapy + from playwright.async_api import async_playwright + + class PlaywrightSpider(scrapy.Spider): + name = "playwright" + start_urls = ["data:,"] # avoid using the default Scrapy downloader + + async def parse(self, response): + async with async_playwright() as pw: + browser = await pw.chromium.launch() + page = await browser.new_page() + await page.goto("https:/example.org") + title = await page.title() + return {"title": title} + + +However, using `playwright-python`_ directly as in the above example +circumvents most of the Scrapy components (middlewares, dupefilter, etc). +We recommend using `scrapy-playwright`_ for a better integration. .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 -.. _chompjs: https://github.com/Nykakin/chompjs .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets +.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript +.. _Splash: https://github.com/scrapinghub/splash +.. _chompjs: https://github.com/Nykakin/chompjs .. _curl: https://curl.haxx.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser -.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript .. _js2xml: https://github.com/scrapinghub/js2xml +.. _playwright-python: https://github.com/microsoft/playwright-python +.. _playwright: https://github.com/microsoft/playwright +.. _pyppeteer: https://pyppeteer.github.io/pyppeteer/ .. _pytesseract: https://github.com/madmaze/pytesseract -.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium +.. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright .. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash -.. _Selenium: https://www.selenium.dev/ -.. _Splash: https://github.com/scrapinghub/splash .. _tabula-py: https://github.com/chezou/tabula-py .. _wget: https://www.gnu.org/software/wget/ -.. _wgrep: https://github.com/stav/wgrep \ No newline at end of file +.. _wgrep: https://github.com/stav/wgrep diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 583a50ab8..9150ca7d9 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -64,10 +64,10 @@ NotConfigured This exception can be raised by some components to indicate that they will remain disabled. Those components include: - * Extensions - * Item pipelines - * Downloader middlewares - * Spider middlewares +- Extensions +- Item pipelines +- Downloader middlewares +- Spider middlewares The exception must be raised in the component's ``__init__`` method. @@ -85,8 +85,8 @@ StopDownload .. exception:: StopDownload(fail=True) -Raised from a :class:`~scrapy.signals.bytes_received` signal handler to -indicate that no further bytes should be downloaded for a response. +Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received` +signal handler to indicate that no further bytes should be downloaded for a response. The ``fail`` boolean parameter controls which method will handle the resulting response: @@ -110,5 +110,6 @@ attribute. ``StopDownload(False)`` or ``StopDownload(True)`` will raise a :class:`TypeError`. -See the documentation for the :class:`~scrapy.signals.bytes_received` signal +See the documentation for the :class:`~scrapy.signals.bytes_received` and +:class:`~scrapy.signals.headers_received` signals and the :ref:`topics-stop-response-download` topic for additional information and examples. diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 6d6bb771d..7580011ac 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -50,18 +50,19 @@ value of one of their fields:: self.year_to_exporter = {} def close_spider(self, spider): - for exporter in self.year_to_exporter.values(): + for exporter, xml_file in self.year_to_exporter.values(): exporter.finish_exporting() + xml_file.close() def _exporter_for_item(self, item): adapter = ItemAdapter(item) year = adapter['year'] if year not in self.year_to_exporter: - f = open(f'{year}.xml', 'wb') - exporter = XmlItemExporter(f) + xml_file = open(f'{year}.xml', 'wb') + exporter = XmlItemExporter(xml_file) exporter.start_exporting() - self.year_to_exporter[year] = exporter - return self.year_to_exporter[year] + self.year_to_exporter[year] = (exporter, xml_file) + return self.year_to_exporter[year][0] def process_item(self, item, spider): exporter = self._exporter_for_item(item) @@ -89,7 +90,7 @@ described next. 1. Declaring a serializer in the field -------------------------------------- -If you use :class:`~.Item` you can declare a serializer in the +If you use :class:`~scrapy.Item` you can declare a serializer in the :ref:`field metadata `. The serializer must be a callable which receives a value and returns its serialized form. @@ -121,9 +122,9 @@ Example:: class ProductXmlExporter(XmlItemExporter): def serialize_field(self, field, name, value): - if field == 'price': + if name == 'price': return f'$ {str(value)}' - return super(Product, self).serialize_field(field, name, value) + return super().serialize_field(field, name, value) .. _topics-exporters-reference: @@ -171,7 +172,7 @@ BaseItemExporter :param field: the field being serialized. If the source :ref:`item object ` does not define field metadata, *field* is an empty :class:`dict`. - :type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance + :type field: :class:`~scrapy.Field` object or a :class:`dict` instance :param name: the name of the field being serialized :type name: str diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 519f18b63..297e1fdc5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -7,8 +7,7 @@ Extensions The extensions framework provides a mechanism for inserting your own custom functionality into Scrapy. -Extensions are just regular classes that are instantiated at Scrapy startup, -when extensions are initialized. +Extensions are just regular classes. Extension settings ================== @@ -27,8 +26,8 @@ Loading & activating extensions =============================== Extensions are loaded and activated at startup by instantiating a single -instance of the extension class. Therefore, all the extension initialization -code must be performed in the class ``__init__`` method. +instance of the extension class per spider being run. All the extension +initialization code must be performed in the class ``__init__`` method. To make an extension available, add it to the :setting:`EXTENSIONS` setting in your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented @@ -324,6 +323,11 @@ domain has finished scraping, including the Scrapy stats collected. The email will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS` setting. +Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a +full list of parameters, including examples on how to instantiate +:class:`~scrapy.mail.MailSender` and use mail settings, see +:ref:`topics-email`. + .. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 6831dcb99..52b47347e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -21,10 +21,10 @@ Serialization formats For serializing the scraped data, the feed exports use the :ref:`Item exporters `. These formats are supported out of the box: - * :ref:`topics-feed-format-json` - * :ref:`topics-feed-format-jsonlines` - * :ref:`topics-feed-format-csv` - * :ref:`topics-feed-format-xml` +- :ref:`topics-feed-format-json` +- :ref:`topics-feed-format-jsonlines` +- :ref:`topics-feed-format-csv` +- :ref:`topics-feed-format-xml` But you can also extend the supported format through the :setting:`FEED_EXPORTERS` setting. @@ -34,54 +34,58 @@ But you can also extend the supported format through the JSON ---- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``json`` - * Exporter used: :class:`~scrapy.exporters.JsonItemExporter` - * See :ref:`this warning ` if you're using JSON with - large feeds. +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``json`` + +- Exporter used: :class:`~scrapy.exporters.JsonItemExporter` + +- See :ref:`this warning ` if you're using JSON with + large feeds. .. _topics-feed-format-jsonlines: JSON lines ---------- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines`` - * Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines`` +- Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter` .. _topics-feed-format-csv: CSV --- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv`` - * Exporter used: :class:`~scrapy.exporters.CsvItemExporter` - * To specify columns to export, their order and their column names, use - :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this - option, but it is important for CSV because unlike many other export - formats CSV uses a fixed header. +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv`` + +- Exporter used: :class:`~scrapy.exporters.CsvItemExporter` + +- To specify columns to export, their order and their column names, use + :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this + option, but it is important for CSV because unlike many other export + formats CSV uses a fixed header. .. _topics-feed-format-xml: XML --- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml`` - * Exporter used: :class:`~scrapy.exporters.XmlItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml`` +- Exporter used: :class:`~scrapy.exporters.XmlItemExporter` .. _topics-feed-format-pickle: Pickle ------ - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle`` - * Exporter used: :class:`~scrapy.exporters.PickleItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle`` +- Exporter used: :class:`~scrapy.exporters.PickleItemExporter` .. _topics-feed-format-marshal: Marshal ------- - * Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal`` - * Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` +- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal`` +- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` .. _topics-feed-storage: @@ -95,11 +99,11 @@ storage backend types which are defined by the URI scheme. The storages backends supported out of the box are: - * :ref:`topics-feed-storage-fs` - * :ref:`topics-feed-storage-ftp` - * :ref:`topics-feed-storage-s3` (requires botocore_) - * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) - * :ref:`topics-feed-storage-stdout` +- :ref:`topics-feed-storage-fs` +- :ref:`topics-feed-storage-ftp` +- :ref:`topics-feed-storage-s3` (requires botocore_) +- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) +- :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are not available. For example, the S3 backend is only available if the botocore_ @@ -114,8 +118,8 @@ Storage URI parameters The storage URI can also contain parameters that get replaced when the feed is being created. These parameters are: - * ``%(time)s`` - gets replaced by a timestamp when the feed is being created - * ``%(name)s`` - gets replaced by the spider name +- ``%(time)s`` - gets replaced by a timestamp when the feed is being created +- ``%(name)s`` - gets replaced by the spider name Any other named parameter gets replaced by the spider attribute of the same name. For example, ``%(site_id)s`` would get replaced by the ``spider.site_id`` @@ -123,13 +127,16 @@ attribute the moment the feed is being created. Here are some examples to illustrate: - * Store in FTP using one directory per spider: +- Store in FTP using one directory per spider: - * ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json`` + - ``ftp://user:password@ftp.example.com/scraping/feeds/%(name)s/%(time)s.json`` - * Store in S3 using one directory per spider: +- Store in S3 using one directory per spider: - * ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` + - ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` + +.. note:: :ref:`Spider arguments ` become spider attributes, hence + they can also be used as storage URI parameters. .. _topics-feed-storage-backends: @@ -144,9 +151,9 @@ Local filesystem The feeds are stored in the local filesystem. - * URI scheme: ``file`` - * Example URI: ``file:///tmp/export.csv`` - * Required external libraries: none +- URI scheme: ``file`` +- Example URI: ``file:///tmp/export.csv`` +- Required external libraries: none Note that for the local filesystem storage (only) you can omit the scheme if you specify an absolute path like ``/tmp/export.csv``. This only works on Unix @@ -159,9 +166,9 @@ FTP The feeds are stored in a FTP server. - * URI scheme: ``ftp`` - * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` - * Required external libraries: none +- URI scheme: ``ftp`` +- Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` +- Required external libraries: none FTP supports two different connection modes: `active or passive `_. Scrapy uses the passive connection @@ -178,23 +185,29 @@ S3 The feeds are stored on `Amazon S3`_. - * URI scheme: ``s3`` - * Example URIs: +- URI scheme: ``s3`` - * ``s3://mybucket/path/to/export.csv`` - * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` +- Example URIs: - * Required external libraries: `botocore`_ >= 1.4.87 + - ``s3://mybucket/path/to/export.csv`` + + - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` + +- Required external libraries: `botocore`_ >= 1.4.87 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: - * :setting:`AWS_ACCESS_KEY_ID` - * :setting:`AWS_SECRET_ACCESS_KEY` +- :setting:`AWS_ACCESS_KEY_ID` +- :setting:`AWS_SECRET_ACCESS_KEY` +- :setting:`AWS_SESSION_TOKEN` (only needed for `temporary security credentials`_) -You can also define a custom ACL for exported feeds using this setting: +.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys - * :setting:`FEED_STORAGE_S3_ACL` +You can also define a custom ACL and custom endpoint for exported feeds using this setting: + +- :setting:`FEED_STORAGE_S3_ACL` +- :setting:`AWS_ENDPOINT_URL` This storage backend uses :ref:`delayed file delivery `. @@ -208,19 +221,20 @@ Google Cloud Storage (GCS) The feeds are stored on `Google Cloud Storage`_. - * URI scheme: ``gs`` - * Example URIs: +- URI scheme: ``gs`` - * ``gs://mybucket/path/to/export.csv`` +- Example URIs: - * Required external libraries: `google-cloud-storage`_. + - ``gs://mybucket/path/to/export.csv`` + +- Required external libraries: `google-cloud-storage`_. For more information about authentication, please refer to `Google Cloud documentation `_. You can set a *Project ID* and *Access Control List (ACL)* through the following settings: - * :setting:`FEED_STORAGE_GCS_ACL` - * :setting:`GCS_PROJECT_ID` +- :setting:`FEED_STORAGE_GCS_ACL` +- :setting:`GCS_PROJECT_ID` This storage backend uses :ref:`delayed file delivery `. @@ -234,9 +248,9 @@ Standard output The feeds are written to the standard output of the Scrapy process. - * URI scheme: ``stdout`` - * Example URI: ``stdout:`` - * Required external libraries: none +- URI scheme: ``stdout`` +- Example URI: ``stdout:`` +- Required external libraries: none .. _delayed-file-delivery: @@ -259,21 +273,118 @@ soon as a file reaches the maximum item count, that file is delivered to the feed URI, allowing item delivery to start way before the end of the crawl. +.. _item-filter: + +Item filtering +============== + +.. versionadded:: 2.6.0 + +You can filter items that you want to allow for a particular feed by using the +``item_classes`` option in :ref:`feeds options `. Only items of +the specified types will be added to the feed. + +The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter` +class, which is the default value of the ``item_filter`` :ref:`feed option `. + +You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s +method ``accepts`` and taking ``feed_options`` as an argument. + +For instance:: + + class MyCustomFilter: + + def __init__(self, feed_options): + self.feed_options = feed_options + + def accepts(self, item): + if "field1" in item and item["field1"] == "expected_data": + return True + return False + + +You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed `. +See :setting:`FEEDS` for examples. + +ItemFilter +---------- + +.. autoclass:: scrapy.extensions.feedexport.ItemFilter + :members: + + +.. _post-processing: + +Post-Processing +=============== + +.. versionadded:: 2.6.0 + +Scrapy provides an option to activate plugins to post-process feeds before they are exported +to feed storages. In addition to using :ref:`builtin plugins `, you +can create your own :ref:`plugins `. + +These plugins can be activated through the ``postprocessing`` option of a feed. +The option must be passed a list of post-processing plugins in the order you want +the feed to be processed. These plugins can be declared either as an import string +or with the imported class of the plugin. Parameters to plugins can be passed +through the feed options. See :ref:`feed options ` for examples. + +.. _builtin-plugins: + +Built-in Plugins +---------------- + +.. autoclass:: scrapy.extensions.postprocessing.GzipPlugin + +.. autoclass:: scrapy.extensions.postprocessing.LZMAPlugin + +.. autoclass:: scrapy.extensions.postprocessing.Bz2Plugin + +.. _custom-plugins: + +Custom Plugins +-------------- + +Each plugin is a class that must implement the following methods: + +.. method:: __init__(self, file, feed_options) + + Initialize the plugin. + + :param file: file-like object having at least the `write`, `tell` and `close` methods implemented + + :param feed_options: feed-specific :ref:`options ` + :type feed_options: :class:`dict` + +.. method:: write(self, data) + + Process and write `data` (:class:`bytes` or :class:`memoryview`) into the plugin's target file. + It must return number of bytes written. + +.. method:: close(self) + + Close the target file object. + +To pass a parameter to your plugin, use :ref:`feed options `. You +can then access those parameters from the ``__init__`` method of your plugin. + + Settings ======== These are the settings used for configuring the feed exports: - * :setting:`FEEDS` (mandatory) - * :setting:`FEED_EXPORT_ENCODING` - * :setting:`FEED_STORE_EMPTY` - * :setting:`FEED_EXPORT_FIELDS` - * :setting:`FEED_EXPORT_INDENT` - * :setting:`FEED_STORAGES` - * :setting:`FEED_STORAGE_FTP_ACTIVE` - * :setting:`FEED_STORAGE_S3_ACL` - * :setting:`FEED_EXPORTERS` - * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` +- :setting:`FEEDS` (mandatory) +- :setting:`FEED_EXPORT_ENCODING` +- :setting:`FEED_STORE_EMPTY` +- :setting:`FEED_EXPORT_FIELDS` +- :setting:`FEED_EXPORT_INDENT` +- :setting:`FEED_STORAGES` +- :setting:`FEED_STORAGE_FTP_ACTIVE` +- :setting:`FEED_STORAGE_S3_ACL` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport @@ -301,21 +412,26 @@ For instance:: 'format': 'json', 'encoding': 'utf8', 'store_empty': False, + 'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'], 'fields': None, 'indent': 4, 'item_export_kwargs': { 'export_empty_fields': True, }, - }, + }, '/home/user/documents/items.xml': { 'format': 'xml', 'fields': ['name', 'price'], + 'item_filter': MyCustomFilter1, 'encoding': 'latin1', 'indent': 8, }, - pathlib.Path('items.csv'): { + pathlib.Path('items.csv.gz'): { 'format': 'csv', 'fields': ['price', 'name'], + 'item_filter': 'myproject.filters.MyCustomFilter2', + 'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 5, }, } @@ -337,6 +453,18 @@ as a fallback value if that key is not provided for a specific feed definition: - ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. +- ``item_classes``: list of :ref:`item classes ` to export. + + If undefined or empty, all items are exported. + + .. versionadded:: 2.6.0 + +- ``item_filter``: a :ref:`filter class ` to filter items to export. + + :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default. + + .. versionadded:: 2.6.0 + - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. - ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. @@ -367,6 +495,11 @@ as a fallback value if that key is not provided for a specific feed definition: - ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. +- ``postprocessing``: list of :ref:`plugins ` to use for post-processing. + + The plugins will be used in the order of the list passed. + + .. versionadded:: 2.6.0 .. setting:: FEED_EXPORT_ENCODING @@ -600,9 +733,12 @@ The function signature should be as follows: :type params: dict :param spider: source spider of the feed items - :type spider: scrapy.spiders.Spider + :type spider: scrapy.Spider -For example, to include the :attr:`name ` of the + .. caution:: The function should return a new dictionary, modifying + the received ``params`` in-place is deprecated. + +For example, to include the :attr:`name ` of the source spider in the feed URI: #. Define the following function somewhere in your project:: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 6287ee0ad..882ff5661 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -42,7 +42,7 @@ Each item pipeline component is a Python class that must implement the following :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Additionally, they may also implement the following methods: @@ -51,18 +51,18 @@ Additionally, they may also implement the following methods: This method is called when the spider is opened. :param spider: the spider which was opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: close_spider(self, spider) This method is called when the spider is closed. :param spider: the spider which was closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object -.. method:: from_crawler(cls, crawler) +.. classmethod:: from_crawler(cls, crawler) - If present, this classmethod is called to create a pipeline instance + If present, this class method is called to create a pipeline instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance of the pipeline. Crawler object provides access to all Scrapy core components like settings and signals; it is a way for pipeline to @@ -190,6 +190,8 @@ item. import scrapy from itemadapter import ItemAdapter + from scrapy.utils.defer import maybe_deferred_to_future + class ScreenshotPipeline: """Pipeline that uses Splash to render screenshot of @@ -202,7 +204,7 @@ item. encoded_item_url = quote(adapter["url"]) screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url) - response = await spider.crawler.engine.download(request, spider) + response = await maybe_deferred_to_future(spider.crawler.engine.download(request, spider)) if response.status != 200: # Error happened, return item. diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 65bf156ac..167014381 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -42,7 +42,8 @@ Item objects :class:`Item` provides a :class:`dict`-like API plus additional features that make it the most feature-complete item type: -.. class:: Item([arg]) +.. class:: scrapy.item.Item([arg]) +.. class:: scrapy.Item([arg]) :class:`Item` objects replicate the standard :class:`dict` API, including its ``__init__`` method. @@ -101,11 +102,6 @@ Additionally, ``dataclass`` items also allow to: * define custom field metadata through :func:`dataclasses.field`, which can be used to :ref:`customize serialization `. -They work natively in Python 3.7 or later, or using the `dataclasses -backport`_ in Python 3.6. - -.. _dataclasses backport: https://pypi.org/project/dataclasses/ - Example:: from dataclasses import dataclass @@ -199,7 +195,8 @@ It's important to note that the :class:`Field` objects used to declare the item do not stay assigned as class attributes. Instead, they can be accessed through the :attr:`Item.fields` attribute. -.. class:: Field([arg]) +.. class:: scrapy.item.Field([arg]) +.. class:: scrapy.Field([arg]) The :class:`Field` class is just an alias to the built-in :class:`dict` class and doesn't provide any extra functionality or attributes. In other words, @@ -317,11 +314,11 @@ If that is not the desired behavior, use a deep copy instead. See :mod:`copy` for more information. To create a shallow copy of an item, you can either call -:meth:`~scrapy.item.Item.copy` on an existing item +:meth:`~scrapy.Item.copy` on an existing item (``product2 = product.copy()``) or instantiate your item class from an existing item (``product2 = Product(product)``). -To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead +To create a deep copy, call :meth:`~scrapy.Item.deepcopy` instead (``product2 = product.deepcopy()``). diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index d855d0133..f16d306c7 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -39,6 +39,8 @@ a signal), and resume it later by issuing the same command:: scrapy crawl somespider -s JOBDIR=crawls/somespider-1 +.. _topics-keeping-persistent-state-between-batches: + Keeping persistent state between batches ======================================== @@ -74,10 +76,10 @@ on cookies. Request serialization --------------------- -For persistence to work, :class:`~scrapy.http.Request` objects must be +For persistence to work, :class:`~scrapy.Request` objects must be serializable with :mod:`pickle`, except for the ``callback`` and ``errback`` values passed to their ``__init__`` method, which must be methods of the -running :class:`~scrapy.spiders.Spider` class. +running :class:`~scrapy.Spider` class. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index b895b95cb..477652704 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -27,7 +27,7 @@ Common causes of memory leaks It happens quite often (sometimes by accident, sometimes on purpose) that the Scrapy developer passes objects referenced in Requests (for example, using the -:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta` +:attr:`~scrapy.Request.cb_kwargs` or :attr:`~scrapy.Request.meta` attributes or the request callback function) and that effectively bounds the lifetime of those referenced objects to the lifetime of the Request. This is, by far, the most common cause of memory leaks in Scrapy projects, and a quite @@ -48,9 +48,9 @@ Too Many Requests? ------------------ By default Scrapy keeps the request queue in memory; it includes -:class:`~scrapy.http.Request` objects and all objects -referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs` -and :attr:`~scrapy.http.Request.meta`). +:class:`~scrapy.Request` objects and all objects +referenced in Request attributes (e.g. in :attr:`~scrapy.Request.cb_kwargs` +and :attr:`~scrapy.Request.meta`). While not necessarily a leak, this can take a lot of memory. Enabling :ref:`persistent job queue ` could help keeping memory usage in control. @@ -90,11 +90,11 @@ Which objects are tracked? The objects tracked by ``trackrefs`` are all from these classes (and all its subclasses): -* :class:`scrapy.http.Request` +* :class:`scrapy.Request` * :class:`scrapy.http.Response` -* :class:`scrapy.item.Item` -* :class:`scrapy.selector.Selector` -* :class:`scrapy.spiders.Spider` +* :class:`scrapy.Item` +* :class:`scrapy.Selector` +* :class:`scrapy.Spider` A real example -------------- diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index c0f534493..0d63700c8 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -56,7 +56,7 @@ chapter `:: l.add_xpath('name', '//div[@class="product_name"]') l.add_xpath('name', '//div[@class="product_title"]') l.add_xpath('price', '//p[@id="price"]') - l.add_css('stock', 'p#stock]') + l.add_css('stock', 'p#stock') l.add_value('last_updated', 'today') # you can also use literal values return l.load_item() diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 55065a1a3..3bf23d5f5 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -93,7 +93,7 @@ path:: Logging from Spiders ==================== -Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider +Scrapy provides a :data:`~scrapy.Spider.logger` within each Spider instance, which can be accessed and used like this:: import scrapy @@ -101,7 +101,7 @@ instance, which can be accessed and used like this:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['https://scrapinghub.com'] + start_urls = ['https://scrapy.org'] def parse(self, response): self.logger.info('Parse function called on %s', response.url) @@ -117,7 +117,7 @@ Python logger you want. For example:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['https://scrapinghub.com'] + start_urls = ['https://scrapy.org'] def parse(self, response): logger.info('Parse function called on %s', response.url) @@ -143,6 +143,7 @@ Logging settings These settings can be used to configure the logging: * :setting:`LOG_FILE` +* :setting:`LOG_FILE_APPEND` * :setting:`LOG_ENABLED` * :setting:`LOG_ENCODING` * :setting:`LOG_LEVEL` @@ -155,7 +156,9 @@ The first couple of settings define a destination for log messages. If :setting:`LOG_FILE` is set, messages sent through the root logger will be redirected to a file named :setting:`LOG_FILE` with encoding :setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log -messages will be displayed on the standard error. Lastly, if +messages will be displayed on the standard error. If :setting:`LOG_FILE` is set +and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten +(discarding the output from previous runs, if any). Lastly, if :setting:`LOG_ENABLED` is ``False``, there won't be any visible log output. :setting:`LOG_LEVEL` determines the minimum level of severity to display, those @@ -215,7 +218,7 @@ For example, let's say you're scraping a website which returns many HTTP 404 and 500 responses, and you want to hide all messages like this:: 2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring - response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code + response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code is not handled or not allowed The first thing to note is a logger name - it is in brackets: @@ -242,6 +245,47 @@ e.g. in the spider's ``__init__`` method:: If you run this spider again then INFO messages from ``scrapy.spidermiddlewares.httperror`` logger will be gone. +You can also filter log records by :class:`~logging.LogRecord` data. For +example, you can filter log records by message content using a substring or +a regular expression. Create a :class:`logging.Filter` subclass +and equip it with a regular expression pattern to +filter out unwanted messages:: + + import logging + import re + + class ContentFilter(logging.Filter): + def filter(self, record): + match = re.search(r'\d{3} [Ee]rror, retrying', record.message) + if match: + return False + +A project-level filter may be attached to the root +handler created by Scrapy, this is a wieldy way to +filter all loggers in different parts of the project +(middlewares, spider, etc.):: + + import logging + import scrapy + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + for handler in logging.root.handlers: + handler.addFilter(ContentFilter()) + +Alternatively, you may choose a specific logger +and hide it without affecting other loggers:: + + import logging + import scrapy + + class MySpider(scrapy.Spider): + # ... + def __init__(self, *args, **kwargs): + logger = logging.getLogger('my_logger') + logger.addFilter(ContentFilter()) + scrapy.utils.log module ======================= diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 156897274..0925e6bb5 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for +The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow @@ -111,25 +111,82 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:: IMAGES_STORE = '/path/to/valid/dir' +.. _topics-file-naming: + +File Naming +=========== + +Default File Naming +------------------- + +By default, files are stored using an `SHA-1 hash`_ of their URLs for the file names. + +For example, the following image URL:: + + http://www.example.com/image.jpg + +Whose ``SHA-1 hash`` is:: + + 3afec3b4765f8f0a07b78f98c07b83f013567a0a + +Will be downloaded and stored using your chosen :ref:`storage method ` and the following file name:: + + 3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg + +Custom File Naming +------------------- + +You may wish to use a different calculated file name for saved files. +For example, classifying an image by including meta in the file name. + +Customize file names by overriding the ``file_path`` method of your +media pipeline. + +For example, an image pipeline with image URL:: + + http://www.example.com/product/images/large/front/0000000004166 + +Can be processed into a file name with a condensed hash and the perspective +``front``:: + + 00b08510e4_front.jpg + +By overriding ``file_path`` like this: + +.. code-block:: python + + import hashlib + from os.path import splitext + + def file_path(self, request, response=None, info=None, *, item=None): + image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5) + image_perspective = request.url.split('/')[-2] + image_filename = f'{image_url_hash}_{image_perspective}.jpg' + + return image_filename + +.. warning:: + If your custom file name scheme relies on meta data that can vary between + scrapes it may lead to unexpected re-downloading of existing media using + new file names. + + For example, if your custom file name scheme uses a product title and the + site changes an item's product title between scrapes, Scrapy will re-download + the same media using updated file names. + +For more information about the ``file_path`` method, see :ref:`topics-media-pipeline-override`. + +.. _topics-supported-storage: + Supported Storage ================= File system storage ------------------- -The files are stored using a `SHA1 hash`_ of their URLs for the file names. +File system storage will save files to the following path:: -For example, the following image URL:: - - http://www.example.com/image.jpg - -Whose ``SHA1 hash`` is:: - - 3afec3b4765f8f0a07b78f98c07b83f013567a0a - -Will be downloaded and stored in the following file:: - - /full/3afec3b4765f8f0a07b78f98c07b83f013567a0a.jpg + /full/ Where: @@ -139,6 +196,9 @@ Where: * ``full`` is a sub-directory to separate full images from thumbnails (if used). For more info see :ref:`topics-images-thumbnails`. +* ```` is the file name assigned to the file. For more info see :ref:`topics-file-naming`. + + .. _media-pipeline-ftp: FTP server storage @@ -259,7 +319,7 @@ respectively), the pipeline will put the results under the respective field When using :ref:`item types ` for which fields are defined beforehand, you must define both the URLs field and the results field. For example, when using the images pipeline, items must define both the ``image_urls`` and the -``images`` field. For instance, using the :class:`~scrapy.item.Item` class:: +``images`` field. For instance, using the :class:`~scrapy.Item` class:: import scrapy @@ -296,6 +356,8 @@ setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used. Additional features =================== +.. _file-expiration: + File expiration --------------- @@ -323,6 +385,9 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key: and pipeline class MyPipeline will have expiration time set to 180. +The last modified time from the file is used to determine the age of the file in days, +which is then compared to the set expiration time to determine if the file is expired. + .. _topics-images-thumbnails: Thumbnail generation for images @@ -353,9 +418,9 @@ Where: * ```` is the one specified in the :setting:`IMAGES_THUMBS` dictionary keys (``small``, ``big``, etc) -* ```` is the `SHA1 hash`_ of the image url +* ```` is the `SHA-1 hash`_ of the image url -.. _SHA1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions +.. _SHA-1 hash: https://en.wikipedia.org/wiki/SHA_hash_functions Example of image files stored using ``small`` and ``big`` thumbnail names:: @@ -424,7 +489,7 @@ See here the methods that you can override in your custom Files Pipeline: In addition to ``response``, this method receives the original :class:`request `, :class:`info ` and - :class:`item ` + :class:`item ` You can override this method to customize the download path of each file. @@ -563,7 +628,7 @@ See here the methods that you can override in your custom Images Pipeline: In addition to ``response``, this method receives the original :class:`request `, :class:`info ` and - :class:`item ` + :class:`item ` You can override this method to customize the download path of each file. @@ -591,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline: .. versionadded:: 2.4 The *item* parameter. + .. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None) + + This method is called for every item of :setting:`IMAGES_THUMBS` per downloaded item. It returns the + thumbnail download path of the image originating from the specified + :class:`response `. + + In addition to ``response``, this method receives the original + :class:`request `, + ``thumb_id``, + :class:`info ` and + :class:`item `. + + You can override this method to customize the thumbnail download path of each image. + You can use the ``item`` to determine the file path based on some item + property. + + By default the :meth:`thumb_path` method returns + ``thumbs//.``. + + .. method:: ImagesPipeline.get_media_requests(item, info) Works the same way as :meth:`FilesPipeline.get_media_requests` method, diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cf1de1bd1..7313c9246 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -63,7 +63,7 @@ project as example. process = CrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. - process.crawl('followall', domain='scrapinghub.com') + process.crawl('followall', domain='scrapy.org') process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling @@ -119,6 +119,7 @@ Here is an example that runs multiple spiders simultaneously: import scrapy from scrapy.crawler import CrawlerProcess + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -128,7 +129,8 @@ Here is an example that runs multiple spiders simultaneously: # Your second spider definition ... - process = CrawlerProcess() + settings = get_project_settings() + process = CrawlerProcess(settings) process.crawl(MySpider1) process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished @@ -141,6 +143,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -151,7 +154,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... configure_logging() - runner = CrawlerRunner() + settings = get_project_settings() + runner = CrawlerRunner(settings) runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() @@ -166,6 +170,7 @@ Same example but running the spiders sequentially by chaining the deferreds: from twisted.internet import reactor, defer from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -175,8 +180,9 @@ Same example but running the spiders sequentially by chaining the deferreds: # Your second spider definition ... - configure_logging() - runner = CrawlerRunner() + settings = get_project_settings() + configure_logging(settings) + runner = CrawlerRunner(settings) @defer.inlineCallbacks def crawl(): @@ -187,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds: crawl() reactor.run() # the script will block here until the last crawl call is finished +Different spiders can set different values for the same setting, but when they +run in the same process it may be impossible, by design or because of some +limitations, to use these different values. What happens in practice is +different for different settings: + +* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value + (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the + default one) cannot be read from the per-spider settings. These are applied + when the :class:`~scrapy.crawler.CrawlerRunner` or + :class:`~scrapy.crawler.CrawlerProcess` object is created. +* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first + available value is used, and if a spider requests a different reactor an + exception will be raised. These are applied when the reactor is installed. +* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the + ones used by the resolver (:setting:`DNSCACHE_ENABLED`, + :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy) + the first available value is used. These are applied when the reactor is + started. + .. seealso:: :ref:`run-from-script`. .. _distributed-crawls: @@ -237,14 +262,14 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use cookies to spot bot behaviour * use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting. -* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites +* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites directly * use a pool of rotating IPs. For example, the free `Tor project`_ or paid services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. * use a highly distributed downloader that circumvents bans internally, so you can just focus on parsing clean pages. One example of such downloaders is - `Crawlera`_ + `Zyte Smart Proxy Manager`_ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. @@ -252,7 +277,7 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Tor project: https://www.torproject.org/ .. _commercial support: https://scrapy.org/support/ .. _ProxyMesh: https://proxymesh.com/ -.. _Google cache: http://www.googleguide.com/cached_pages.html +.. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders -.. _Crawlera: https://scrapinghub.com/crawlera .. _scrapoxy: https://scrapoxy.io/ +.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f3aaa2c8f..49cb69f67 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -26,10 +26,6 @@ Request objects .. autoclass:: Request - A :class:`Request` object represents an HTTP request, which is usually - generated in the Spider and executed by the Downloader, and thus generating - a :class:`Response`. - :param url: the URL of this request If the URL is invalid, a :exc:`ValueError` exception is raised. @@ -39,7 +35,7 @@ Request objects request (once it's downloaded) as its first parameter. For more information see :ref:`topics-request-response-ref-request-callback-arguments` below. If a Request doesn't specify a callback, the spider's - :meth:`~scrapy.spiders.Spider.parse` method will be used. + :meth:`~scrapy.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. :type callback: collections.abc.Callable @@ -64,7 +60,7 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. :type headers: dict @@ -96,7 +92,7 @@ Request objects To create a request that does not send stored cookies and does not store received cookies, set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta `. + in :attr:`request.meta `. Example of a request that sends manually-defined cookies and ignores cookie storage:: @@ -111,9 +107,13 @@ Request objects .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. + .. versionadded:: 2.6.0 + Cookie values that are :class:`bool`, :class:`float` or :class:`int` + are casted to :class:`str`. + :type cookies: dict or list :param encoding: the encoding of this request (defaults to ``'utf-8'``). @@ -205,6 +205,8 @@ Request objects ``failure.request.cb_kwargs`` in the request's errback. For more information, see :ref:`errback-cb_kwargs`. + .. autoattribute:: Request.attributes + .. method:: Request.copy() Return a new Request which is a copy of this Request. See also: @@ -220,6 +222,15 @@ Request objects .. automethod:: from_curl + .. automethod:: to_dict + + +Other functions related to requests +----------------------------------- + +.. autofunction:: scrapy.utils.request.request_from_dict + + .. _topics-request-response-ref-request-callback-arguments: Passing additional data to callback functions @@ -293,7 +304,7 @@ errors if needed:: "http://www.httpbin.org/status/404", # Not found error "http://www.httpbin.org/status/500", # server issue "http://www.httpbin.org:12345/", # non-responding host, timeout expected - "http://www.httphttpbinbin.org/", # DNS error expected + "https://example.invalid/", # DNS error expected ] def start_requests(self): @@ -328,6 +339,7 @@ errors if needed:: request = failure.request self.logger.error('TimeoutError on %s', request.url) + .. _errback-cb_kwargs: Accessing additional data in errback functions @@ -353,6 +365,273 @@ achieve this by using ``Failure.request.cb_kwargs``:: main_url=failure.request.cb_kwargs['main_url'], ) + +.. _request-fingerprints: + +Request fingerprints +-------------------- + +There are some aspects of scraping, such as filtering out duplicate requests +(see :setting:`DUPEFILTER_CLASS`) or caching responses (see +:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short, +unique identifier from a :class:`~scrapy.http.Request` object: a request +fingerprint. + +You often do not need to worry about request fingerprints, the default request +fingerprinter works for most projects. + +However, there is no universal way to generate a unique identifier from a +request, because different situations require comparing requests differently. +For example, sometimes you may need to compare URLs case-insensitively, include +URL fragments, exclude certain URL query parameters, include some or all +headers, etc. + +To change how request fingerprints are built for your requests, use the +:setting:`REQUEST_FINGERPRINTER_CLASS` setting. + +.. setting:: REQUEST_FINGERPRINTER_CLASS + +REQUEST_FINGERPRINTER_CLASS +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: VERSION + +Default: :class:`scrapy.utils.request.RequestFingerprinter` + +A :ref:`request fingerprinter class ` or its +import path. + +.. autoclass:: scrapy.utils.request.RequestFingerprinter + + +.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION + +REQUEST_FINGERPRINTER_IMPLEMENTATION +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: VERSION + +Default: ``'PREVIOUS_VERSION'`` + +Determines which request fingerprinting algorithm is used by the default +request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). + +Possible values are: + +- ``'PREVIOUS_VERSION'`` (default) + + This implementation uses the same request fingerprinting algorithm as + Scrapy PREVIOUS_VERSION and earlier versions. + + Even though this is the default value for backward compatibility reasons, + it is a deprecated value. + +- ``'VERSION'`` + + This implementation was introduced in Scrapy VERSION to fix an issue of the + previous implementation. + + New projects should use this value. The :command:`startproject` command + sets this value in the generated ``settings.py`` file. + +If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are +using Scrapy components where changing the request fingerprinting algorithm +would cause undesired results, you need to carefully decide when to change the +value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS` +setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request +fingerprinting algorithm and does not log this warning ( +:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a +class). + +Scenarios where changing the request fingerprinting algorithm may cause +undesired results include, for example, using the HTTP cache middleware (see +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). +Changing the request fingerprinting algorithm would invalidade the current +cache, requiring you to redownload all requests again. + +Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in +your settings to switch already to the request fingerprinting implementation +that will be the only request fingerprinting implementation available in a +future version of Scrapy, and remove the deprecation warning triggered by using +the default value (``'PREVIOUS_VERSION'``). + + +.. _PREVIOUS_VERSION-request-fingerprinter: +.. _custom-request-fingerprinter: + +Writing your own request fingerprinter +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A request fingerprinter is a class that must implement the following method: + +.. method:: fingerprint(self, request) + + Return a :class:`bytes` object that uniquely identifies *request*. + + See also :ref:`request-fingerprint-restrictions`. + + :param request: request to fingerprint + :type request: scrapy.http.Request + +Additionally, it may also implement the following methods: + +.. classmethod:: from_crawler(cls, crawler) + + If present, this class method is called to create a request fingerprinter + instance from a :class:`~scrapy.crawler.Crawler` object. It must return a + new instance of the request fingerprinter. + + *crawler* provides access to all Scrapy core components like settings and + signals; it is a way for the request fingerprinter to access them and hook + its functionality into Scrapy. + + :param crawler: crawler that uses this request fingerprinter + :type crawler: :class:`~scrapy.crawler.Crawler` object + +.. classmethod:: from_settings(cls, settings) + + If present, and ``from_crawler`` is not defined, this class method is called + to create a request fingerprinter instance from a + :class:`~scrapy.settings.Settings` object. It must return a new instance of + the request fingerprinter. + +The ``fingerprint`` method of the default request fingerprinter, +:class:`scrapy.utils.request.RequestFingerprinter`, uses +:func:`scrapy.utils.request.fingerprint` with its default parameters. For some +common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well +in your ``fingerprint`` method implementation: + +.. autofunction:: scrapy.utils.request.fingerprint + +For example, to take the value of a request header named ``X-ID`` into +account:: + + # my_project/settings.py + REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter' + + # my_project/utils.py + from scrapy.utils.request import fingerprint + + class RequestFingerprinter: + + def fingerprint(self, request): + return fingerprint(request, include_headers=['X-ID']) + +You can also write your own fingerprinting logic from scratch. + +However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure +you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints: + +- Caching saves CPU by ensuring that fingerprints are calculated only once + per request, and not once per Scrapy component that needs the fingerprint + of a request. + +- Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that + request objects do not stay in memory forever just because you have + references to them in your cache dictionary. + +For example, to take into account only the URL of a request, without any prior +URL canonicalization or taking the request method or body into account:: + + from hashlib import sha1 + from weakref import WeakKeyDictionary + + from scrapy.utils.python import to_bytes + + class RequestFingerprinter: + + cache = WeakKeyDictionary() + + def fingerprint(self, request): + if request not in self.cache: + fp = sha1() + fp.update(to_bytes(request.url)) + self.cache[request] = fp.digest() + return self.cache[request] + +If you need to be able to override the request fingerprinting for arbitrary +requests from your spider callbacks, you may implement a request fingerprinter +that reads fingerprints from :attr:`request.meta ` +when available, and then falls back to +:func:`~scrapy.utils.request.fingerprint`. For example:: + + from scrapy.utils.request import fingerprint + + class RequestFingerprinter: + + def fingerprint(self, request): + if 'fingerprint' in request.meta: + return request.meta['fingerprint'] + return fingerprint(request) + +If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION +without using the deprecated ``'PREVIOUS_VERSION'`` value of the +:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following +request fingerprinter:: + + from hashlib import sha1 + from weakref import WeakKeyDictionary + + from scrapy.utils.python import to_bytes + from w3lib.url import canonicalize_url + + class RequestFingerprinter: + + cache = WeakKeyDictionary() + + def fingerprint(self, request): + if request not in self.cache: + fp = sha1() + fp.update(to_bytes(request.method)) + fp.update(to_bytes(canonicalize_url(request.url))) + fp.update(request.body or b'') + self.cache[request] = fp.digest() + return self.cache[request] + + +.. _request-fingerprint-restrictions: + +Request fingerprint restrictions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Scrapy components that use request fingerprints may impose additional +restrictions on the format of the fingerprints that your :ref:`request +fingerprinter ` generates. + +The following built-in Scrapy components have such restrictions: + +- :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default + value of :setting:`HTTPCACHE_STORAGE`) + + Request fingerprints must be at least 1 byte long. + + Path and filename length limits of the file system of + :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`, + the following directory structure is created: + + - :attr:`Spider.name ` + + - first byte of a request fingerprint as hexadecimal + + - fingerprint as hexadecimal + + - filenames up to 16 characters long + + For example, if a request fingerprint is made of 20 bytes (default), + :setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``, + and the name of your spider is ``'my_spider'`` your file system must + support a file path like:: + + /home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers + +- :class:`scrapy.extensions.httpcache.DbmCacheStorage` + + The underlying DBM implementation must support keys as long as twice + the number of bytes of a request fingerprint, plus 5. For example, + if a request fingerprint is made of 20 bytes (default), + 45-character-long keys must be supported. + + .. _topics-request-meta: Request.meta special keys @@ -363,26 +642,26 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: -* :reqmeta:`dont_redirect` -* :reqmeta:`dont_retry` -* :reqmeta:`handle_httpstatus_list` -* :reqmeta:`handle_httpstatus_all` -* :reqmeta:`dont_merge_cookies` +* :reqmeta:`bindaddress` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` +* :reqmeta:`dont_merge_cookies` +* :reqmeta:`dont_obey_robotstxt` +* :reqmeta:`dont_redirect` +* :reqmeta:`dont_retry` +* :reqmeta:`download_fail_on_dataloss` +* :reqmeta:`download_latency` +* :reqmeta:`download_maxsize` +* :reqmeta:`download_timeout` +* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* ``ftp_user`` (See :setting:`FTP_USER` for more info) +* :reqmeta:`handle_httpstatus_all` +* :reqmeta:`handle_httpstatus_list` +* :reqmeta:`max_retry_times` +* :reqmeta:`proxy` * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` -* :reqmeta:`bindaddress` -* :reqmeta:`dont_obey_robotstxt` -* :reqmeta:`download_timeout` -* :reqmeta:`download_maxsize` -* :reqmeta:`download_latency` -* :reqmeta:`download_fail_on_dataloss` -* :reqmeta:`proxy` -* ``ftp_user`` (See :setting:`FTP_USER` for more info) -* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * :reqmeta:`referrer_policy` -* :reqmeta:`max_retry_times` .. reqmeta:: bindaddress @@ -432,9 +711,9 @@ The meta key is used set retry times per request. When initialized, the Stopping the download of a Response =================================== -Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a -:class:`~scrapy.signals.bytes_received` signal handler will stop the -download of a given response. See the following example:: +Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the +:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received` +signals will stop the download of a given response. See the following example:: import scrapy @@ -488,7 +767,9 @@ fields with form data from :class:`Response` objects. .. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms -.. class:: FormRequest(url, [formdata, ...]) +.. class:: scrapy.http.request.form.FormRequest +.. class:: scrapy.http.FormRequest +.. class:: scrapy.FormRequest(url, [formdata, ...]) The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The remaining arguments are the same as for the :class:`Request` class and are @@ -642,6 +923,8 @@ dealing with JSON requests. data into JSON format. :type dumps_kwargs: dict + .. autoattribute:: JsonRequest.attributes + JsonRequest usage example ------------------------- @@ -659,9 +942,6 @@ Response objects .. autoclass:: Response - A :class:`Response` object represents an HTTP response, which is usually - downloaded (by the Downloader) and fed to the Spiders for processing. - :param url: the URL of this response :type url: str @@ -685,7 +965,7 @@ Response objects :param request: the initial value of the :attr:`Response.request` attribute. This represents the :class:`Request` that generated this response. - :type request: scrapy.http.Request + :type request: scrapy.Request :param certificate: an object representing the server's SSL certificate. :type certificate: twisted.internet.ssl.Certificate @@ -693,9 +973,19 @@ Response objects :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` + :param protocol: The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1", "h2" + :type protocol: :class:`str` + + .. versionadded:: 2.0.0 + The ``certificate`` parameter. + .. versionadded:: 2.1.0 The ``ip_address`` parameter. + .. versionadded:: 2.5.0 + The ``protocol`` parameter. + .. attribute:: Response.url A string containing the URL of the response. @@ -780,6 +1070,8 @@ Response objects .. attribute:: Response.certificate + .. versionadded:: 2.0.0 + A :class:`twisted.internet.ssl.Certificate` object representing the server's SSL certificate. @@ -795,6 +1087,19 @@ Response objects handler, i.e. for ``http(s)`` responses. For other handlers, :attr:`ip_address` is always ``None``. + .. attribute:: Response.protocol + + .. versionadded:: 2.5.0 + + The protocol that was used to download the response. + For instance: "HTTP/1.0", "HTTP/1.1" + + This attribute is currently only populated by the HTTP download + handlers, i.e. for ``http(s)`` responses. For other handlers, + :attr:`protocol` is always ``None``. + + .. autoattribute:: Response.attributes + .. method:: Response.copy() Returns a new Response which is a copy of this Response. @@ -888,9 +1193,11 @@ TextResponse objects .. attribute:: TextResponse.selector - A :class:`~scrapy.selector.Selector` instance using the response as + A :class:`~scrapy.Selector` instance using the response as target. The selector is lazily instantiated on first access. + .. autoattribute:: TextResponse.attributes + :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: @@ -915,6 +1222,14 @@ TextResponse objects Returns a Python object from deserialized JSON document. The result is cached after the first call. + .. method:: TextResponse.urljoin(url) + + Constructs an absolute url by combining the Response's base url with + a possible relative url. The base url shall be extracted from the + ```` tag, or just the Response's :attr:`url` if there is no such + tag. + + HtmlResponse objects -------------------- diff --git a/docs/topics/scheduler.rst b/docs/topics/scheduler.rst new file mode 100644 index 000000000..57c24b76a --- /dev/null +++ b/docs/topics/scheduler.rst @@ -0,0 +1,34 @@ +.. _topics-scheduler: + +========= +Scheduler +========= + +.. module:: scrapy.core.scheduler + +The scheduler component receives requests from the :ref:`engine ` +and stores them into persistent and/or non-persistent data structures. +It also gets those requests and feeds them back to the engine when it +asks for a next request to be downloaded. + + +Overriding the default scheduler +================================ + +You can use your own custom scheduler class by supplying its full +Python path in the :setting:`SCHEDULER` setting. + + +Minimal scheduler interface +=========================== + +.. autoclass:: BaseScheduler + :members: + + +Default Scrapy scheduler +======================== + +.. autoclass:: Scheduler + :members: + :special-members: __len__ diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index b576fde91..574d4568c 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -8,14 +8,14 @@ When you're scraping web pages, the most common task you need to perform is to extract data from the HTML source. There are several libraries available to achieve this, such as: - * `BeautifulSoup`_ is a very popular web scraping library among Python - programmers which constructs a Python object based on the structure of the - HTML code and also deals with bad markup reasonably well, but it has one - drawback: it's slow. +- `BeautifulSoup`_ is a very popular web scraping library among Python + programmers which constructs a Python object based on the structure of the + HTML code and also deals with bad markup reasonably well, but it has one + drawback: it's slow. - * `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic - API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard - library.) +- `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic + API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python + standard library.) Scrapy comes with its own mechanism for extracting data. They're called selectors because they "select" certain parts of the HTML document specified @@ -48,7 +48,7 @@ Constructing selectors .. highlight:: python -Response objects expose a :class:`~scrapy.selector.Selector` instance +Response objects expose a :class:`~scrapy.Selector` instance on ``.selector`` attribute: >>> response.selector.xpath('//span/text()').get() @@ -62,7 +62,7 @@ more shortcuts: ``response.xpath()`` and ``response.css()``: >>> response.css('span::text').get() 'good' -Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class +Scrapy selectors are instances of :class:`~scrapy.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or markup as a string (in ``text`` argument). @@ -175,7 +175,7 @@ of ``None``: 'not-found' Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes -using ``.attrib`` property of a :class:`~scrapy.selector.Selector`: +using ``.attrib`` property of a :class:`~scrapy.Selector`: >>> [img.attrib['src'] for img in response.css('img')] ['image1_thumb.jpg', @@ -383,7 +383,7 @@ ID, or when selecting an unique element on a page): Using selectors with regular expressions ---------------------------------------- -:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting +:class:`~scrapy.Selector` also has a ``.re()`` method for extracting data using regular expressions. However, unlike using ``.xpath()`` or ``.css()`` methods, ``.re()`` returns a list of strings. So you can't construct nested ``.re()`` calls. @@ -464,10 +464,10 @@ effectively. If you are not much familiar with XPath yet, you may want to take a look first at this `XPath tutorial`_. .. note:: - Some of the tips are based on `this post from ScrapingHub's blog`_. + Some of the tips are based on `this post from Zyte's blog`_. .. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html -.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/ +.. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/ .. _topics-selectors-relative-xpaths: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0086a6c74..2046c6446 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -67,7 +67,7 @@ Example:: Spiders (See the :ref:`topics-spiders` chapter for reference) can define their own settings that will take precedence and override the project ones. They can -do so by setting their :attr:`~scrapy.spiders.Spider.custom_settings` attribute:: +do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute:: class MySpider(scrapy.Spider): name = 'myspider' @@ -142,7 +142,7 @@ In a spider, the settings are available through ``self.settings``:: The ``settings`` attribute is set in the base Spider class after the spider is initialized. If you want to use the settings before the initialization (e.g., in your spider's ``__init__()`` method), you'll need to override the - :meth:`~scrapy.spiders.Spider.from_crawler` method. + :meth:`~scrapy.Spider.from_crawler` method. Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` attribute of the Crawler that is passed to ``from_crawler`` method in @@ -204,6 +204,19 @@ Default: ``None`` The AWS secret key used by code that requires access to `Amazon Web services`_, such as the :ref:`S3 feed storage backend `. +.. setting:: AWS_SESSION_TOKEN + +AWS_SESSION_TOKEN +----------------- + +Default: ``None`` + +The AWS security token used by code that requires access to `Amazon Web services`_, +such as the :ref:`S3 feed storage backend `, when using +`temporary security credentials`_. + +.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys + .. setting:: AWS_ENDPOINT_URL AWS_ENDPOINT_URL @@ -338,7 +351,7 @@ is non-zero, download delay is enforced per IP, not per domain. DEFAULT_ITEM_CLASS ------------------ -Default: ``'scrapy.item.Item'`` +Default: ``'scrapy.Item'`` The default class that will be used for instantiating items in the :ref:`the Scrapy shell `. @@ -360,7 +373,7 @@ The default headers used for Scrapy HTTP Requests. They're populated in the .. caution:: Cookies set via the ``Cookie`` header are not considered by the :ref:`cookies-mw`. If you need to set cookies for a request, use the - :class:`Request.cookies ` parameter. This is a known + :class:`Request.cookies ` parameter. This is a known current limitation that is being worked on. .. setting:: DEPTH_LIMIT @@ -384,8 +397,8 @@ Default: ``0`` Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware`` -An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of -a :class:`~scrapy.http.Request` based on its depth. +An integer that is used to adjust the :attr:`~scrapy.Request.priority` of +a :class:`~scrapy.Request` based on its depth. The priority of a request is adjusted as follows:: @@ -657,6 +670,7 @@ DOWNLOAD_HANDLERS_BASE Default:: { + 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', @@ -677,6 +691,45 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } +.. _http2: + +The default HTTPS handler uses HTTP/1.1. To use HTTP/2: + +#. Install ``Twisted[http2]>=17.9.0`` to install the packages required to + enable HTTP/2 support in Twisted. + +#. Update :setting:`DOWNLOAD_HANDLERS` as follows:: + + DOWNLOAD_HANDLERS = { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', + } + +.. warning:: + + HTTP/2 support in Scrapy is experimental, and not yet recommended for + production environments. Future Scrapy versions may introduce related + changes without a deprecation period or warning. + +.. note:: + + Known limitations of the current HTTP/2 implementation of Scrapy include: + + - No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). + + - No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will + fail. + + - No support for `server pushes`_, which are ignored. + + - No support for the :signal:`bytes_received` and + :signal:`headers_received` signals. + +.. _frame size: https://tools.ietf.org/html/rfc7540#section-4.2 +.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption +.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT @@ -754,6 +807,15 @@ Optionally, this can be set per-request basis by using the If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. +.. warning:: + + This setting is ignored by the + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` + download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss + error, the corresponding HTTP/2 connection may be corrupted, affecting other + requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])`` + failure is always raised for every request that was using that connection. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS @@ -763,18 +825,14 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'`` The class used to detect and filter duplicate requests. -The default (``RFPDupeFilter``) filters based on request fingerprint using -the ``scrapy.utils.request.request_fingerprint`` function. In order to change -the way duplicates are checked you could subclass ``RFPDupeFilter`` and -override its ``request_fingerprint`` method. This method should accept -scrapy :class:`~scrapy.http.Request` object and return its fingerprint -(a string). +The default (``RFPDupeFilter``) filters based on the +:setting:`REQUEST_FINGERPRINTER_CLASS` setting. You can disable filtering of duplicate requests by setting :setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. Be very careful about this however, because you can get into crawling loops. It's usually a better idea to set the ``dont_filter`` parameter to -``True`` on the specific :class:`~scrapy.http.Request` that should not be +``True`` on the specific :class:`~scrapy.Request` that should not be filtered. .. setting:: DUPEFILTER_DEBUG @@ -927,6 +985,16 @@ Default: ``{}`` A dict containing the pipelines enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. +.. setting:: JOBDIR + +JOBDIR +------ + +Default: ``''`` + +A string indicating the directory for storing the state of a crawl when +:ref:`pausing and resuming crawls `. + .. setting:: LOG_ENABLED LOG_ENABLED @@ -954,6 +1022,16 @@ Default: ``None`` File name to use for logging output. If ``None``, standard error will be used. +.. setting:: LOG_FILE_APPEND + +LOG_FILE_APPEND +--------------- + +Default: ``True`` + +If ``False``, the log file specified with :setting:`LOG_FILE` will be +overwritten (discarding the output from previous runs, if any). + .. setting:: LOG_FORMAT LOG_FORMAT @@ -1188,20 +1266,6 @@ Adjust redirect request priority relative to original request: - **a positive priority adjust (default) means higher priority.** - a negative priority adjust means lower priority. -.. setting:: RETRY_PRIORITY_ADJUST - -RETRY_PRIORITY_ADJUST ---------------------- - -Default: ``-1`` - -Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware`` - -Adjust retry request priority relative to original request: - -- a positive priority adjust means higher priority. -- **a negative priority adjust (default) means lower priority.** - .. setting:: ROBOTSTXT_OBEY ROBOTSTXT_OBEY @@ -1249,7 +1313,8 @@ SCHEDULER Default: ``'scrapy.core.scheduler.Scheduler'`` -The scheduler to use for crawling. +The scheduler class to be used for crawling. +See the :ref:`topics-scheduler` topic for details. .. setting:: SCHEDULER_DEBUG @@ -1507,7 +1572,7 @@ If a reactor is already installed, :meth:`CrawlerRunner.__init__ ` raises :exc:`Exception` if the installed reactor does not match the -:setting:`TWISTED_REACTOR` setting; therfore, having top-level +:setting:`TWISTED_REACTOR` setting; therefore, having top-level :mod:`~twisted.internet.reactor` imports in project files and imported third-party libraries will make Scrapy raise :exc:`Exception` when it checks which reactor is installed. @@ -1528,7 +1593,7 @@ In order to use the reactor installed by Scrapy:: def start_requests(self): reactor.callLater(self.timeout, self.stop) - urls = ['http://quotes.toscrape.com/page/1'] + urls = ['https://quotes.toscrape.com/page/1'] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -1556,7 +1621,7 @@ which raises :exc:`Exception`, becomes:: from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) - urls = ['http://quotes.toscrape.com/page/1'] + urls = ['https://quotes.toscrape.com/page/1'] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -1569,10 +1634,9 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will not attempt to install any specific reactor, and the -default reactor defined by Twisted for the current platform will be used. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +means that Scrapy will install the default reactor defined by Twisted for the +current platform. This is to maintain backward compatibility and avoid possible +problems caused by using a non-default reactor. For additional information, see :doc:`core/howto/choosing-reactor`. @@ -1586,8 +1650,19 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` -The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html +The maximum URL length to allow for crawled URLs. + +This setting can act as a stopping condition in case of URLs of ever-increasing +length, which may be caused for example by a programming error either in the +target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and +:setting:`DEPTH_LIMIT`. + +Use ``0`` to allow URLs of any length. + +The default value is copied from the `Microsoft Internet Explorer maximum URL +length`_, even though this setting exists for different reasons. + +.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT @@ -1599,7 +1674,7 @@ Default: ``"Scrapy/VERSION (+https://scrapy.org)"`` The default User-Agent to use when crawling, unless overridden. This user agent is also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and -there is no overridding User-Agent header specified for the request. +there is no overriding User-Agent header specified for the request. Settings documented elsewhere: @@ -1610,7 +1685,6 @@ case to see how to enable and use them. .. settingslist:: - .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 0f46f1c87..007e9fc2f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -95,20 +95,21 @@ convenience. Available Shortcuts ------------------- - * ``shelp()`` - print a help with the list of available objects and shortcuts +- ``shelp()`` - print a help with the list of available objects and + shortcuts - * ``fetch(url[, redirect=True])`` - fetch a new response from the given - URL and update all related objects accordingly. You can optionaly ask for - HTTP 3xx redirections to not be followed by passing ``redirect=False`` +- ``fetch(url[, redirect=True])`` - fetch a new response from the given URL + and update all related objects accordingly. You can optionally ask for HTTP + 3xx redirections to not be followed by passing ``redirect=False`` - * ``fetch(request)`` - fetch a new response from the given request and - update all related objects accordingly. +- ``fetch(request)`` - fetch a new response from the given request and update + all related objects accordingly. - * ``view(response)`` - open the given response in your local web browser, for - inspection. This will add a `\ tag`_ to the response body in order - for external links (such as images and style sheets) to display properly. - Note, however, that this will create a temporary file in your computer, - which won't be removed automatically. +- ``view(response)`` - open the given response in your local web browser, for + inspection. This will add a `\ tag`_ to the response body in order + for external links (such as images and style sheets) to display properly. + Note, however, that this will create a temporary file in your computer, + which won't be removed automatically. .. _ tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base @@ -117,26 +118,26 @@ Available Scrapy objects The Scrapy shell automatically creates some convenient objects from the downloaded page, like the :class:`~scrapy.http.Response` object and the -:class:`~scrapy.selector.Selector` objects (for both HTML and XML +:class:`~scrapy.Selector` objects (for both HTML and XML content). Those objects are: - * ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. +- ``crawler`` - the current :class:`~scrapy.crawler.Crawler` object. - * ``spider`` - the Spider which is known to handle the URL, or a - :class:`~scrapy.spiders.Spider` object if there is no spider found for - the current URL +- ``spider`` - the Spider which is known to handle the URL, or a + :class:`~scrapy.Spider` object if there is no spider found for the + current URL - * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched - page. You can modify this request using :meth:`~scrapy.http.Request.replace` - or fetch a new request (without leaving the shell) using the ``fetch`` - shortcut. +- ``request`` - a :class:`~scrapy.Request` object of the last fetched + page. You can modify this request using + :meth:`~scrapy.Request.replace` or fetch a new request (without + leaving the shell) using the ``fetch`` shortcut. - * ``response`` - a :class:`~scrapy.http.Response` object containing the last - fetched page +- ``response`` - a :class:`~scrapy.http.Response` object containing the last + fetched page - * ``settings`` - the current :ref:`Scrapy settings ` +- ``settings`` - the current :ref:`Scrapy settings ` Example of shell session ======================== diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 1d99d8c28..17bd16156 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -51,16 +51,16 @@ Deferred signal handlers ======================== Some signals support returning :class:`~twisted.internet.defer.Deferred` -objects from their handlers, allowing you to run asynchronous code that -does not block Scrapy. If a signal handler returns a -:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that -:class:`~twisted.internet.defer.Deferred` to fire. +or :term:`awaitable objects ` from their handlers, allowing +you to run asynchronous code that does not block Scrapy. If a signal +handler returns one of these objects, Scrapy waits for that asynchronous +operation to finish. -Let's take an example:: +Let's take an example using :ref:`coroutines `:: class SignalSpider(scrapy.Spider): name = 'signals' - start_urls = ['http://quotes.toscrape.com/page/1/'] + start_urls = ['https://quotes.toscrape.com/page/1/'] @classmethod def from_crawler(cls, crawler, *args, **kwargs): @@ -68,17 +68,15 @@ Let's take an example:: crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped) return spider - def item_scraped(self, item): + async def item_scraped(self, item): # Send the scraped item to the server - d = treq.post( + response = await treq.post( 'http://example.com/post', json.dumps(item).encode('ascii'), headers={b'Content-Type': [b'application/json']} ) - # The next item will be scraped only after - # deferred (d) is fired - return d + return response def parse(self, response): for quote in response.css('div.quote'): @@ -89,7 +87,7 @@ Let's take an example:: } See the :ref:`topics-signals-ref` below to know which signals support -:class:`~twisted.internet.defer.Deferred`. +:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects `. .. _topics-signals-ref: @@ -155,7 +153,7 @@ item_scraped :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was scraped :type response: :class:`~scrapy.http.Response` object @@ -175,7 +173,7 @@ item_dropped :type item: :ref:`item object ` :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param response: the response from where the item was dropped :type response: :class:`~scrapy.http.Response` object @@ -203,7 +201,7 @@ item_error :type response: :class:`~scrapy.http.Response` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -223,7 +221,7 @@ spider_closed This signal supports returning deferreds from its handlers. :param spider: the spider which has been closed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object :param reason: a string which describes the reason why the spider was closed. If it was closed because the spider has completed scraping, the reason @@ -247,7 +245,7 @@ spider_opened This signal supports returning deferreds from its handlers. :param spider: the spider which has been opened - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object spider_idle ~~~~~~~~~~~ @@ -268,10 +266,17 @@ spider_idle You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to prevent the spider from being closed. + Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider` + exception to provide a custom spider closing reason. An + idle handler is the perfect place to put some code that assesses + the final spider results and update the final closing reason + accordingly (e.g. setting it to 'too_few_results' instead of + 'finished'). + This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. note:: Scheduling some requests in your :signal:`spider_idle` handler does **not** guarantee that it can prevent the spider from being closed, @@ -296,7 +301,7 @@ spider_error :type response: :class:`~scrapy.http.Response` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object Request signals --------------- @@ -307,16 +312,16 @@ request_scheduled .. signal:: request_scheduled .. function:: request_scheduled(request, spider) - Sent when the engine schedules a :class:`~scrapy.http.Request`, to be + Sent when the engine schedules a :class:`~scrapy.Request`, to be downloaded later. This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_dropped ~~~~~~~~~~~~~~~ @@ -324,16 +329,16 @@ request_dropped .. signal:: request_dropped .. function:: request_dropped(request, spider) - Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be + Sent when a :class:`~scrapy.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_reached_downloader ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -341,15 +346,15 @@ request_reached_downloader .. signal:: request_reached_downloader .. function:: request_reached_downloader(request, spider) - Sent when a :class:`~scrapy.http.Request` reached downloader. + Sent when a :class:`~scrapy.Request` reached downloader. This signal does not support returning deferreds from its handlers. :param request: the request that reached downloader - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object request_left_downloader ~~~~~~~~~~~~~~~~~~~~~~~ @@ -359,16 +364,16 @@ request_left_downloader .. versionadded:: 2.0 - Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of + Sent when a :class:`~scrapy.Request` leaves the downloader, even in case of failure. This signal does not support returning deferreds from its handlers. :param request: the request that reached the downloader - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object bytes_received ~~~~~~~~~~~~~~ @@ -384,22 +389,52 @@ bytes_received a possible scenario for a 25 kb response would be two signals fired with 10 kb of data, and a final one with 5 kb of data. + Handlers for this signal can stop the download of a response while it + is in progress by raising the :exc:`~scrapy.exceptions.StopDownload` + exception. Please refer to the :ref:`topics-stop-response-download` topic + for additional information and examples. + This signal does not support returning deferreds from its handlers. :param data: the data received by the download handler :type data: :class:`bytes` object :param request: the request that generated the download - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider associated with the response - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object -.. note:: Handlers of this signal can stop the download of a response while it +headers_received +~~~~~~~~~~~~~~~~ + +.. versionadded:: 2.5 + +.. signal:: headers_received +.. function:: headers_received(headers, body_length, request, spider) + + Sent by the HTTP 1.1 and S3 download handlers when the response headers are + available for a given request, before downloading any additional content. + + Handlers for this signal can stop the download of a response while it is in progress by raising the :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the :ref:`topics-stop-response-download` topic for additional information and examples. + This signal does not support returning deferreds from its handlers. + + :param headers: the headers received by the download handler + :type headers: :class:`scrapy.http.headers.Headers` object + + :param body_length: expected size of the response body, in bytes + :type body_length: `int` + + :param request: the request that generated the download + :type request: :class:`~scrapy.Request` object + + :param spider: the spider associated with the response + :type spider: :class:`~scrapy.Spider` object + Response signals ---------------- @@ -418,10 +453,10 @@ response_received :type response: :class:`~scrapy.http.Response` object :param request: the request that generated the response - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. note:: The ``request`` argument might not contain the original request that reached the downloader, if a :ref:`topics-downloader-middleware` modifies @@ -442,7 +477,7 @@ response_downloaded :type response: :class:`~scrapy.http.Response` object :param request: the request that generated the response - :type request: :class:`~scrapy.http.Request` object + :type request: :class:`~scrapy.Request` object :param spider: the spider for which the response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index fc114a63f..f27bc79c0 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -93,7 +93,7 @@ object gives you access, for example, to the :ref:`settings `. :type response: :class:`~scrapy.http.Response` object :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_output(response, result, spider) @@ -102,7 +102,7 @@ object gives you access, for example, to the :ref:`settings `. it has processed the response. :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request` objects and :ref:`item object + :class:`~scrapy.Request` objects and :ref:`item object `. :param response: the response which generated this output from the @@ -110,11 +110,11 @@ object gives you access, for example, to the :ref:`settings `. :type response: :class:`~scrapy.http.Response` object :param result: the result returned by the spider - :type result: an iterable of :class:`~scrapy.http.Request` objects and + :type result: an iterable of :class:`~scrapy.Request` objects and :ref:`item object ` :param spider: the spider whose result is being processed - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_spider_exception(response, exception, spider) @@ -122,8 +122,8 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Request` objects and :ref:`item object - `. + iterable of :class:`~scrapy.Request` or :ref:`item ` + objects. If it returns ``None``, Scrapy will continue processing this exception, executing any other :meth:`process_spider_exception` in the following @@ -142,7 +142,7 @@ object gives you access, for example, to the :ref:`settings `. :type exception: :exc:`Exception` object :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: process_start_requests(start_requests, spider) @@ -152,7 +152,7 @@ object gives you access, for example, to the :ref:`settings `. items). It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.http.Request` objects. + return another iterable of :class:`~scrapy.Request` objects. .. note:: When implementing this method in your spider middleware, you should always return an iterable (that follows the input one) and @@ -164,10 +164,10 @@ object gives you access, for example, to the :ref:`settings `. (like a time limit or item/page count). :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.http.Request` + :type start_requests: an iterable of :class:`~scrapy.Request` :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.spiders.Spider` object + :type spider: :class:`~scrapy.Spider` object .. method:: from_crawler(cls, crawler) @@ -251,9 +251,10 @@ this:: .. reqmeta:: handle_httpstatus_all The ``handle_httpstatus_list`` key of :attr:`Request.meta -` can also be used to specify which response codes to +` can also be used to specify which response codes to allow on a per-request basis. You can also set the meta key ``handle_httpstatus_all`` -to ``True`` if you want to allow any response code for a request. +to ``True`` if you want to allow any response code for a request, and ``False`` to +disable the effects of the ``handle_httpstatus_all`` key. Keep in mind, however, that it's usually a bad idea to handle non-200 responses, unless you really know what you're doing. @@ -294,7 +295,7 @@ OffsiteMiddleware Filters out Requests for URLs outside the domains covered by the spider. This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute. + spider's :attr:`~scrapy.Spider.allowed_domains` attribute. All subdomains of any domain in the list are also allowed. E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` but not ``www2.example.com`` nor ``example.com``. @@ -312,10 +313,10 @@ OffsiteMiddleware will be printed (but only for the first request filtered). If the spider doesn't define an - :attr:`~scrapy.spiders.Spider.allowed_domains` attribute, or the + :attr:`~scrapy.Spider.allowed_domains` attribute, or the attribute is empty, the offsite middleware will allow all requests. - If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute + If the request has the :attr:`~scrapy.Request.dont_filter` attribute set, the offsite middleware will allow the request even if its domain is not listed in allowed domains. @@ -439,4 +440,3 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. - diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 2056664c7..ece02ae47 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -17,15 +17,15 @@ For spiders, the scraping cycle goes through something like this: those requests. The first requests to perform are obtained by calling the - :meth:`~scrapy.spiders.Spider.start_requests` method which (by default) - generates :class:`~scrapy.http.Request` for the URLs specified in the - :attr:`~scrapy.spiders.Spider.start_urls` and the - :attr:`~scrapy.spiders.Spider.parse` method as callback function for the + :meth:`~scrapy.Spider.start_requests` method which (by default) + generates :class:`~scrapy.Request` for the URLs specified in the + :attr:`~scrapy.Spider.start_urls` and the + :attr:`~scrapy.Spider.parse` method as callback function for the Requests. 2. In the callback function, you parse the response (web page) and return :ref:`item objects `, - :class:`~scrapy.http.Request` objects, or an iterable of these objects. + :class:`~scrapy.Request` objects, or an iterable of these objects. Those Requests will also contain a callback (maybe the same) and will then be downloaded by Scrapy and then their response handled by the specified callback. @@ -42,15 +42,13 @@ Even though this cycle applies (more or less) to any kind of spider, there are different kinds of default spiders bundled into Scrapy for different purposes. We will talk about those types here. -.. module:: scrapy.spiders - :synopsis: Spiders base class, spider manager and spider middleware - .. _topics-spiders-ref: scrapy.Spider ============= -.. class:: Spider() +.. class:: scrapy.spiders.Spider +.. class:: scrapy.Spider() This is the simplest spider, and the one from which every other spider must inherit (including spiders that come bundled with Scrapy, as well as spiders @@ -86,7 +84,7 @@ scrapy.Spider A list of URLs where the spider will begin to crawl from, when no particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data + listed here. The subsequent :class:`~scrapy.Request` will be generated successively from data contained in the start URLs. .. attribute:: custom_settings @@ -121,6 +119,11 @@ scrapy.Spider send log messages through it as described on :ref:`topics-logging-from-spiders`. + .. attribute:: state + + A dict you can use to persist some spider state between batches. + See :ref:`topics-keeping-persistent-state-between-batches` to know more about it. + .. method:: from_crawler(crawler, *args, **kwargs) This is the class method used by Scrapy to create your spiders. @@ -179,7 +182,7 @@ scrapy.Spider the same requirements as the :class:`Spider` class. This method, as well as any other Request callback, must return an - iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects + iterable of :class:`~scrapy.Request` and/or :ref:`item objects `. :param response: the response to parse @@ -234,7 +237,7 @@ Return multiple Requests and items from a single callback:: yield scrapy.Request(response.urljoin(href), self.parse) Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; -to give data more structure you can use :class:`~scrapy.item.Item` objects:: +to give data more structure you can use :class:`~scrapy.Item` objects:: import scrapy from myproject.items import MyItem @@ -294,6 +297,14 @@ The above example can also be written as follows:: def start_requests(self): yield scrapy.Request(f'http://www.example.com/categories/{self.category}') +If you are :ref:`running Scrapy from a script `, you can +specify spider arguments when calling +:class:`CrawlerProcess.crawl ` or +:class:`CrawlerRunner.crawl `:: + + process = CrawlerProcess() + process.crawl(MySpider, category="electronics") + Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. If you were to set the ``start_urls`` attribute from the command line, @@ -358,14 +369,14 @@ CrawlSpider described below. If multiple rules match the same link, the first one will be used, according to the order they're defined in this attribute. - This spider also exposes an overrideable method: + This spider also exposes an overridable method: .. method:: parse_start_url(response, **kwargs) This method is called for each response produced for the URLs in the spider's ``start_urls`` attribute. It allows to parse the initial responses and must return either an - :ref:`item object `, a :class:`~scrapy.http.Request` + :ref:`item object `, a :class:`~scrapy.Request` object, or an iterable containing any of them. Crawling rules @@ -375,7 +386,7 @@ Crawling rules ``link_extractor`` is a :ref:`Link Extractor ` object which defines how links will be extracted from each crawled page. Each produced link will - be used to generate a :class:`~scrapy.http.Request` object, which will contain the + be used to generate a :class:`~scrapy.Request` object, which will contain the link's text in its ``meta`` dictionary (under the ``link_text`` key). If omitted, a default link extractor created with no arguments will be used, resulting in all links being extracted. @@ -384,9 +395,9 @@ Crawling rules object with that name will be used) to be called for each link extracted with the specified link extractor. This callback receives a :class:`~scrapy.http.Response` as its first argument and must return either a single instance or an iterable of - :ref:`item objects ` and/or :class:`~scrapy.http.Request` objects + :ref:`item objects ` and/or :class:`~scrapy.Request` objects (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response` - object will contain the text of the link that produced the :class:`~scrapy.http.Request` + object will contain the text of the link that produced the :class:`~scrapy.Request` in its ``meta`` dictionary (under the ``link_text`` key) ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the @@ -403,7 +414,7 @@ Crawling rules ``process_request`` is a callable (or a string, in which case a method from the spider object with that name will be used) which will be called for every - :class:`~scrapy.http.Request` extracted by this rule. This callable should + :class:`~scrapy.Request` extracted by this rule. This callable should take said request as first argument and the :class:`~scrapy.http.Response` from which the request originated as second argument. It must return a ``Request`` object or ``None`` (to filter out the request). @@ -414,10 +425,9 @@ Crawling rules It receives a :class:`Twisted Failure ` instance as first parameter. - -.. warning:: Because of its internal implementation, you must explicitly set - callbacks for new requests when writing :class:`CrawlSpider`-based spiders; - unexpected behaviour can occur otherwise. + .. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`CrawlSpider`-based spiders; + unexpected behaviour can occur otherwise. .. versionadded:: 2.0 The *errback* parameter. @@ -463,7 +473,7 @@ Let's now take a look at an example CrawlSpider with rules:: This spider would start crawling example.com's home page, collecting category links, and item links, parsing the latter with the ``parse_item`` method. For each item response, some data will be extracted from the HTML using XPath, and -an :class:`~scrapy.item.Item` will be filled with it. +an :class:`~scrapy.Item` will be filled with it. XMLFeedSpider ------------- @@ -486,11 +496,11 @@ XMLFeedSpider - ``'iternodes'`` - a fast iterator based on regular expressions - - ``'html'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + - ``'html'`` - an iterator which uses :class:`~scrapy.Selector`. Keep in mind this uses DOM parsing and must load all DOM in memory which could be a problem for big feeds - - ``'xml'`` - an iterator which uses :class:`~scrapy.selector.Selector`. + - ``'xml'`` - an iterator which uses :class:`~scrapy.Selector`. Keep in mind this uses DOM parsing and must load all DOM in memory which could be a problem for big feeds @@ -508,7 +518,7 @@ XMLFeedSpider available in that document that will be processed with this spider. The ``prefix`` and ``uri`` will be used to automatically register namespaces using the - :meth:`~scrapy.selector.Selector.register_namespace` method. + :meth:`~scrapy.Selector.register_namespace` method. You can then specify nodes with namespaces in the :attr:`itertag` attribute. @@ -521,7 +531,7 @@ XMLFeedSpider itertag = 'n:url' # ... - Apart from these new attributes, this spider has the following overrideable + Apart from these new attributes, this spider has the following overridable methods too: .. method:: adapt_response(response) @@ -535,10 +545,10 @@ XMLFeedSpider This method is called for the nodes matching the provided tag name (``itertag``). Receives the response and an - :class:`~scrapy.selector.Selector` for each node. Overriding this + :class:`~scrapy.Selector` for each node. Overriding this method is mandatory. Otherwise, you spider won't work. This method must return an :ref:`item object `, a - :class:`~scrapy.http.Request` object, or an iterable containing any of + :class:`~scrapy.Request` object, or an iterable containing any of them. .. method:: process_results(response, results) @@ -549,10 +559,9 @@ XMLFeedSpider item IDs. It receives a list of results and the response which originated those results. It must return a list of results (items or requests). - -.. warning:: Because of its internal implementation, you must explicitly set - callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; - unexpected behaviour can occur otherwise. + .. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; + unexpected behaviour can occur otherwise. XMLFeedSpider example @@ -581,7 +590,7 @@ These spiders are pretty easy to use, let's have a look at one example:: Basically what we did up there was to create a spider that downloads a feed from the given ``start_urls``, and then iterates through each of its ``item`` tags, -prints them out, and stores some random data in an :class:`~scrapy.item.Item`. +prints them out, and stores some random data in an :class:`~scrapy.Item`. CSVFeedSpider ------------- diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 9802a34a2..832829b75 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -110,11 +110,10 @@ using the telnet console:: Execution engine status time()-engine.start_time : 8.62972998619 - engine.has_capacity() : False len(engine.downloader.active) : 16 engine.scraper.is_idle() : False engine.spider.name : followall - engine.spider_is_idle(engine.spider) : False + engine.spider_is_idle() : False engine.slot.closing : False len(engine.slot.inprogress) : 16 len(engine.slot.scheduler.dqs or []) : 0 diff --git a/docs/versioning.rst b/docs/versioning.rst index 57643ea9a..9d02757b0 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -13,7 +13,7 @@ There are 3 numbers in a Scrapy version: *A.B.C* large changes. * *B* is the release number. This will include many changes including features and things that possibly break backward compatibility, although we strive to - keep theses cases at a minimum. + keep these cases at a minimum. * *C* is the bugfix release number. Backward-incompatibilities are explicitly mentioned in the :ref:`release notes `, diff --git a/extras/qpsclient.py b/extras/qpsclient.py index f9fb70342..28703650d 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -1,5 +1,5 @@ """ -A spider that generate light requests to meassure QPS troughput +A spider that generate light requests to meassure QPS throughput usage: diff --git a/pylintrc b/pylintrc index 5b6b9fab0..2cdd6321e 100644 --- a/pylintrc +++ b/pylintrc @@ -6,6 +6,7 @@ jobs=1 # >1 hides results disable=abstract-method, anomalous-backslash-in-string, arguments-differ, + arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, bad-continuation, @@ -21,9 +22,12 @@ disable=abstract-method, cell-var-from-loop, comparison-with-callable, consider-iterating-dictionary, + consider-using-dict-items, + consider-using-from-import, consider-using-in, consider-using-set-comprehension, consider-using-sys-exit, + consider-using-with, cyclic-import, dangerous-default-value, deprecated-method, @@ -101,11 +105,14 @@ disable=abstract-method, unnecessary-lambda, unnecessary-pass, unreachable, + unspecified-encoding, unsubscriptable-object, unused-argument, unused-import, + unused-private-member, unused-variable, unused-wildcard-import, + use-implicit-booleaness-not-comparison, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, diff --git a/pytest.ini b/pytest.ini index 1c95f715a..ae2ed2029 100644 --- a/pytest.ini +++ b/pytest.ini @@ -18,26 +18,6 @@ addopts = --ignore=docs/topics/stats.rst --ignore=docs/topics/telnetconsole.rst --ignore=docs/utils -twisted = 1 markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed -flake8-max-line-length = 119 -flake8-ignore = - W503 - - # Exclude files that are meant to provide top-level imports - # E402: Module level import not at top of file - # F401: Module imported but unused - scrapy/__init__.py E402 - scrapy/core/downloader/handlers/http.py F401 - scrapy/http/__init__.py F401 - scrapy/linkextractors/__init__.py E402 F401 - scrapy/selector/__init__.py F401 - scrapy/spiders/__init__.py E402 F401 - - # Issues pending a review: - scrapy/utils/http.py F403 - scrapy/utils/markup.py F403 - scrapy/utils/multipart.py F403 - scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 + only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed diff --git a/scrapy/VERSION b/scrapy/VERSION index 005119baa..6a6a3d8e3 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.4.1 +2.6.1 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 4326ca4aa..86e584396 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -22,14 +22,14 @@ __all__ = [ # Scrapy and Twisted versions -__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip() +__version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").strip() version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.')) twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 6): - print("Scrapy %s requires Python 3.6+" % __version__) +if sys.version_info < (3, 7): + print(f"Scrapy {__version__} requires Python 3.7+") sys.exit(1) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 91482ce01..491c4beab 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,13 +1,13 @@ import sys import os -import optparse +import argparse import cProfile import inspect import pkg_resources import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.commands import ScrapyCommand +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings @@ -123,8 +123,6 @@ def execute(argv=None, settings=None): inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) - parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), - conflict_handler='resolve') if not cmdname: _print_commands(settings, inproject) sys.exit(0) @@ -133,12 +131,14 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser.usage = f"scrapy {cmdname} {cmd.syntax()}" - parser.description = cmd.long_desc() + parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, + usage=f"scrapy {cmdname} {cmd.syntax()}", + conflict_handler='resolve', + description=cmd.long_desc()) settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings cmd.add_options(parser) - opts, args = parser.parse_args(args=argv[1:]) + opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) cmd.crawler_process = CrawlerProcess(settings) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 23ccffcd9..fb304b8c0 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -2,7 +2,9 @@ Base class for Scrapy commands """ import os -from optparse import OptionGroup +import argparse +from typing import Any, Dict + from twisted.python import failure from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli @@ -15,7 +17,7 @@ class ScrapyCommand: crawler_process = None # default settings to be used for this command instead of global defaults - default_settings = {} + default_settings: Dict[str, Any] = {} exitcode = 0 @@ -41,14 +43,14 @@ class ScrapyCommand: def long_desc(self): """A long description of the command. Return short description when not - available. It cannot contain newlines, since contents will be formatted + available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() def help(self): """An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines, since no post-formatting will + "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() @@ -57,22 +59,20 @@ class ScrapyCommand: """ Populate option parse with options available for this command """ - group = OptionGroup(parser, "Global Options") - group.add_option("--logfile", metavar="FILE", - help="log file. if omitted stderr will be used") - group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help=f"log level (default: {self.settings['LOG_LEVEL']})") - group.add_option("--nolog", action="store_true", - help="disable logging completely") - group.add_option("--profile", metavar="FILE", default=None, - help="write python cProfile stats to FILE") - group.add_option("--pidfile", metavar="FILE", - help="write process ID to FILE") - group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", - help="set/override setting (may be repeated)") - group.add_option("--pdb", action="store_true", help="enable pdb on failure") - - parser.add_option_group(group) + group = parser.add_argument_group(title='Global Options') + group.add_argument("--logfile", metavar="FILE", + help="log file. if omitted stderr will be used") + group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None, + help=f"log level (default: {self.settings['LOG_LEVEL']})") + group.add_argument("--nolog", action="store_true", + help="disable logging completely") + group.add_argument("--profile", metavar="FILE", default=None, + help="write python cProfile stats to FILE") + group.add_argument("--pidfile", metavar="FILE", + help="write process ID to FILE") + group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE", + help="set/override setting (may be repeated)") + group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args, opts): try: @@ -112,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand): """ def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="append scraped items to the end of FILE (use - for stdout)") - parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append", - help="dump scraped items into FILE, overwriting any existing file") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items") + parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_argument("-o", "--output", metavar="FILE", action="append", + help="append scraped items to the end of FILE (use - for stdout)") + parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append", + help="dump scraped items into FILE, overwriting any existing file") + parser.add_argument("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -135,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand): opts.overwrite_output, ) self.settings.set('FEEDS', feeds, priority='cmdline') + + +class ScrapyHelpFormatter(argparse.HelpFormatter): + """ + Help Formatter for scrapy command line help messages. + """ + def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): + super().__init__(prog, indent_increment=indent_increment, + max_help_position=max_help_position, width=width) + + def _join_parts(self, part_strings): + parts = self.format_part_strings(part_strings) + return super()._join_parts(parts) + + def format_part_strings(self, part_strings): + """ + Underline and title case command line help message headers. + """ + if part_strings and part_strings[0].startswith("usage: "): + part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):] + headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')] + for index in headings[::-1]: + char = '-' if "Global Options" in part_strings[index] else '=' + part_strings[index] = part_strings[index][:-2].title() + underline = ''.join(["\n", (char * len(part_strings[index])), "\n"]) + part_strings.insert(index + 1, underline) + return part_strings diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 999c987ea..6bdf9eae0 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider): def start_requests(self): qargs = {'total': self.total, 'show': self.show} - url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}' + url = f'{self.baseurl}?{urlencode(qargs, doseq=True)}' return [scrapy.Request(url, dont_filter=True)] def parse(self, response): diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index ae21d86e6..a16f4beb7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -49,10 +49,10 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-l", "--list", dest="list", action="store_true", - help="only list contracts, without checking them") - parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true', - help="print contract tests for all spiders") + parser.add_argument("-l", "--list", dest="list", action="store_true", + help="only list contracts, without checking them") + parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true', + help="print contract tests for all spiders") def run(self, args, opts): # load contracts diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index f205c40b0..0f2a21b85 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand): if len(args) < 1: raise UsageError() elif len(args) > 1: - raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") + raise UsageError("running 'scrapy crawl' with more than one spider is not supported") spname = args[0] crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 95f87e8c3..9b2ebb37f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -26,11 +26,11 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", help="use this spider") - parser.add_option("--headers", dest="headers", action="store_true", - help="print response HTTP headers instead of body") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument("--spider", dest="spider", help="use this spider") + parser.add_argument("--headers", dest="headers", action="store_true", + help="print response HTTP headers instead of body") + parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 5f44daa70..ed5f588e9 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,6 +4,7 @@ import string from importlib import import_module from os.path import join, dirname, abspath, exists, splitext +from urllib.parse import urlparse import scrapy from scrapy.commands import ScrapyCommand @@ -22,6 +23,14 @@ def sanitize_module_name(module_name): return module_name +def extract_domain(url): + """Extract domain name from URL string""" + o = urlparse(url) + if o.scheme == '' and o.netloc == '': + o = urlparse("//" + url.lstrip("/")) + return o.netloc + + class Command(ScrapyCommand): requires_project = False @@ -35,16 +44,16 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-l", "--list", dest="list", action="store_true", - help="List available templates") - parser.add_option("-e", "--edit", dest="edit", action="store_true", - help="Edit spider after creating it") - parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE", - help="Dump template to standard output") - parser.add_option("-t", "--template", dest="template", default="basic", - help="Uses a custom template.") - parser.add_option("--force", dest="force", action="store_true", - help="If the spider already exists, overwrite it with the template") + parser.add_argument("-l", "--list", dest="list", action="store_true", + help="List available templates") + parser.add_argument("-e", "--edit", dest="edit", action="store_true", + help="Edit spider after creating it") + parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE", + help="Dump template to standard output") + parser.add_argument("-t", "--template", dest="template", default="basic", + help="Uses a custom template.") + parser.add_argument("--force", dest="force", action="store_true", + help="If the spider already exists, overwrite it with the template") def run(self, args, opts): if opts.list: @@ -59,7 +68,8 @@ class Command(ScrapyCommand): if len(args) != 2: raise UsageError() - name, domain = args[0:2] + name, url = args[0:2] + domain = extract_domain(url) module = sanitize_module_name(name) if self.settings.get('BOT_NAME') == module: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 83ee074da..99fc8f955 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,5 +1,6 @@ import json import logging +from typing import Dict from itemadapter import is_item, ItemAdapter from w3lib.url import is_url @@ -10,6 +11,7 @@ from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError + logger = logging.getLogger(__name__) @@ -17,8 +19,8 @@ class Command(BaseRunSpiderCommand): requires_project = True spider = None - items = {} - requests = {} + items: Dict[int, list] = {} + requests: Dict[int, list] = {} first_response = None @@ -30,28 +32,28 @@ class Command(BaseRunSpiderCommand): def add_options(self, parser): BaseRunSpiderCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", default=None, - help="use this spider without looking for one") - parser.add_option("--pipelines", action="store_true", - help="process items through pipelines") - parser.add_option("--nolinks", dest="nolinks", action="store_true", - help="don't show links to follow (extracted requests)") - parser.add_option("--noitems", dest="noitems", action="store_true", - help="don't show scraped items") - parser.add_option("--nocolour", dest="nocolour", action="store_true", - help="avoid using pygments to colorize the output") - parser.add_option("-r", "--rules", dest="rules", action="store_true", - help="use CrawlSpider rules to discover the callback") - parser.add_option("-c", "--callback", dest="callback", - help="use this callback for parsing, instead looking for a callback") - parser.add_option("-m", "--meta", dest="meta", - help="inject extra meta into the Request, it must be a valid raw json string") - parser.add_option("--cbkwargs", dest="cbkwargs", - help="inject extra callback kwargs into the Request, it must be a valid raw json string") - parser.add_option("-d", "--depth", dest="depth", type="int", default=1, - help="maximum depth for parsing requests [default: %default]") - parser.add_option("-v", "--verbose", dest="verbose", action="store_true", - help="print each depth level one by one") + parser.add_argument("--spider", dest="spider", default=None, + help="use this spider without looking for one") + parser.add_argument("--pipelines", action="store_true", + help="process items through pipelines") + parser.add_argument("--nolinks", dest="nolinks", action="store_true", + help="don't show links to follow (extracted requests)") + parser.add_argument("--noitems", dest="noitems", action="store_true", + help="don't show scraped items") + parser.add_argument("--nocolour", dest="nocolour", action="store_true", + help="avoid using pygments to colorize the output") + parser.add_argument("-r", "--rules", dest="rules", action="store_true", + help="use CrawlSpider rules to discover the callback") + parser.add_argument("-c", "--callback", dest="callback", + help="use this callback for parsing, instead looking for a callback") + parser.add_argument("-m", "--meta", dest="meta", + help="inject extra meta into the Request, it must be a valid raw json string") + parser.add_argument("--cbkwargs", dest="cbkwargs", + help="inject extra callback kwargs into the Request, it must be a valid raw json string") + parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, + help="maximum depth for parsing requests [default: %default]") + parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", + help="print each depth level one by one") @property def max_level(self): @@ -144,7 +146,8 @@ class Command(BaseRunSpiderCommand): def _start_requests(spider): yield self.prepare_request(spider, Request(url), opts) - self.spidercls.start_requests = _start_requests + if self.spidercls: + self.spidercls.start_requests = _start_requests def start_parsing(self, url, opts): self.crawler_process.crawl(self.spidercls, **opts.spargs) diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 8d49e440f..1b2e2601e 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -18,16 +18,16 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--get", dest="get", metavar="SETTING", - help="print raw setting value") - parser.add_option("--getbool", dest="getbool", metavar="SETTING", - help="print setting value, interpreted as a boolean") - parser.add_option("--getint", dest="getint", metavar="SETTING", - help="print setting value, interpreted as an integer") - parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", - help="print setting value, interpreted as a float") - parser.add_option("--getlist", dest="getlist", metavar="SETTING", - help="print setting value, interpreted as a list") + parser.add_argument("--get", dest="get", metavar="SETTING", + help="print raw setting value") + parser.add_argument("--getbool", dest="getbool", metavar="SETTING", + help="print setting value, interpreted as a boolean") + parser.add_argument("--getint", dest="getint", metavar="SETTING", + help="print setting value, interpreted as an integer") + parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING", + help="print setting value, interpreted as a float") + parser.add_argument("--getlist", dest="getlist", metavar="SETTING", + help="print setting value, interpreted as a list") def run(self, args, opts): settings = self.crawler_process.settings diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index d1944df3d..f67a5886a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -33,12 +33,12 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-c", dest="code", - help="evaluate the code in the shell, print the result and exit") - parser.add_option("--spider", dest="spider", - help="use this spider") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument("-c", dest="code", + help="evaluate the code in the shell, print the result and exit") + parser.add_argument("--spider", dest="spider", + help="use this spider") + parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -75,6 +75,6 @@ class Command(ScrapyCommand): def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, - kwargs={'stop_after_crawl': False}) + kwargs={'stop_after_crawl': False, 'install_signal_handlers': False}) t.daemon = True t.start() diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 1d73fa0cb..1b6374c39 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,7 +1,7 @@ import re import os import string -from importlib import import_module +from importlib.util import find_spec from os.path import join, exists, abspath from shutil import ignore_patterns, move, copy2, copystat from stat import S_IWUSR as OWNER_WRITE_PERMISSION @@ -42,11 +42,8 @@ class Command(ScrapyCommand): def _is_valid_name(self, project_name): def _module_exists(module_name): - try: - import_module(module_name) - return True - except ImportError: - return False + spec = find_spec(module_name) + return spec is not None and spec.loader is not None if not re.search(r'^[_a-zA-Z]\w*$', project_name): print('Error: Project names must begin with a letter and contain' diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 1237610cb..c6a3c273a 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -16,8 +16,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--verbose", "-v", dest="verbose", action="store_true", - help="also display twisted/python/platform info (useful for bug reports)") + parser.add_argument("--verbose", "-v", dest="verbose", action="store_true", + help="also display twisted/python/platform info (useful for bug reports)") def run(self, args, opts): if opts.verbose: diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index c8f873334..b1f52abe2 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,3 +1,4 @@ +import argparse from scrapy.commands import fetch from scrapy.utils.response import open_in_browser @@ -12,7 +13,7 @@ class Command(fetch.Command): def add_options(self, parser): super().add_options(parser) - parser.remove_option("--headers") + parser.add_argument('--headers', help=argparse.SUPPRESS) def _print_response(self, response, opts): open_in_browser(response) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index db0a56e56..b47e55092 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -1,16 +1,77 @@ -import sys import re +import sys from functools import wraps from inspect import getmembers +from typing import Dict from unittest import TestCase from scrapy.http import Request -from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec +from scrapy.utils.spider import iterate_spider_output + + +class Contract: + """ Abstract class for contracts """ + request_cls = None + + def __init__(self, method, *args): + self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') + self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') + self.args = args + + def add_pre_hook(self, request, results): + if hasattr(self, 'pre_process'): + cb = request.callback + + @wraps(cb) + def wrapper(response, **cb_kwargs): + try: + results.startTest(self.testcase_pre) + self.pre_process(response) + results.stopTest(self.testcase_pre) + except AssertionError: + results.addFailure(self.testcase_pre, sys.exc_info()) + except Exception: + results.addError(self.testcase_pre, sys.exc_info()) + else: + results.addSuccess(self.testcase_pre) + finally: + return list(iterate_spider_output(cb(response, **cb_kwargs))) + + request.callback = wrapper + + return request + + def add_post_hook(self, request, results): + if hasattr(self, 'post_process'): + cb = request.callback + + @wraps(cb) + def wrapper(response, **cb_kwargs): + output = list(iterate_spider_output(cb(response, **cb_kwargs))) + try: + results.startTest(self.testcase_post) + self.post_process(output) + results.stopTest(self.testcase_post) + except AssertionError: + results.addFailure(self.testcase_post, sys.exc_info()) + except Exception: + results.addError(self.testcase_post, sys.exc_info()) + else: + results.addSuccess(self.testcase_post) + finally: + return output + + request.callback = wrapper + + return request + + def adjust_request_args(self, args): + return args class ContractsManager: - contracts = {} + contracts: Dict[str, Contract] = {} def __init__(self, contracts): for contract in contracts: @@ -107,66 +168,6 @@ class ContractsManager: request.errback = eb_wrapper -class Contract: - """ Abstract class for contracts """ - request_cls = None - - def __init__(self, method, *args): - self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') - self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') - self.args = args - - def add_pre_hook(self, request, results): - if hasattr(self, 'pre_process'): - cb = request.callback - - @wraps(cb) - def wrapper(response, **cb_kwargs): - try: - results.startTest(self.testcase_pre) - self.pre_process(response) - results.stopTest(self.testcase_pre) - except AssertionError: - results.addFailure(self.testcase_pre, sys.exc_info()) - except Exception: - results.addError(self.testcase_pre, sys.exc_info()) - else: - results.addSuccess(self.testcase_pre) - finally: - return list(iterate_spider_output(cb(response, **cb_kwargs))) - - request.callback = wrapper - - return request - - def add_post_hook(self, request, results): - if hasattr(self, 'post_process'): - cb = request.callback - - @wraps(cb) - def wrapper(response, **cb_kwargs): - output = list(iterate_spider_output(cb(response, **cb_kwargs))) - try: - results.startTest(self.testcase_post) - self.post_process(output) - results.stopTest(self.testcase_post) - except AssertionError: - results.addFailure(self.testcase_post, sys.exc_info()) - except Exception: - results.addError(self.testcase_post, sys.exc_info()) - else: - results.addSuccess(self.testcase_post) - finally: - return output - - request.callback = wrapper - - return request - - def adjust_request_args(self, args): - return args - - def _create_testcase(method, desc): spider = method.__self__.name diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 8a7d656a1..b5318c7bb 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,10 +1,15 @@ +import warnings + from OpenSSL import SSL +from twisted.internet._sslverify import _setAcceptableProtocols from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer +from zope.interface.verify import verifyObject -from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS +from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions +from scrapy.utils.misc import create_instance, load_object @implementer(IPolicyForHTTPS) @@ -81,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): The default OpenSSL method is ``TLS_METHOD`` (also called ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname, port): # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -92,3 +97,51 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): trustRoot=platformTrust(), extraCertificateOptions={'method': self._ssl_method}, ) + + +@implementer(IPolicyForHTTPS) +class AcceptableProtocolsContextFactory: + """Context factory to used to override the acceptable protocols + to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN + negotiation. + """ + + def __init__(self, context_factory, acceptable_protocols): + verifyObject(IPolicyForHTTPS, context_factory) + self._wrapped_context_factory = context_factory + self._acceptable_protocols = acceptable_protocols + + def creatorForNetloc(self, hostname, port): + options = self._wrapped_context_factory.creatorForNetloc(hostname, port) + _setAcceptableProtocols(options._ctx, self._acceptable_protocols) + return options + + +def load_context_factory_from_settings(settings, crawler): + ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] + context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + # try method-aware context factory + try: + context_factory = create_instance( + objcls=context_factory_cls, + settings=settings, + crawler=crawler, + method=ssl_method, + ) + except TypeError: + # use context factory defaults + context_factory = create_instance( + objcls=context_factory_cls, + settings=settings, + crawler=crawler, + ) + msg = ( + f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept " + "a `method` argument (type OpenSSL.SSL method, e.g. " + "OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` " + "argument and/or a `tls_ciphers` argument. Please, upgrade your " + "context factory class to handle them or ignore them." + ) + warnings.warn(msg) + + return context_factory diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1b041c8a8..38935667d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,7 +7,7 @@ import warnings from contextlib import suppress from io import BytesIO from time import time -from urllib.parse import urldefrag +from urllib.parse import urldefrag, urlunparse from twisted.internet import defer, protocol, ssl from twisted.internet.endpoints import TCP4ClientEndpoint @@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from zope.interface import implementer from scrapy import signals -from scrapy.core.downloader.tls import openssl_methods +from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload from scrapy.http import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import to_bytes, to_unicode @@ -43,29 +42,7 @@ class HTTP11DownloadHandler: self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False - self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] - self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - # try method-aware context factory - try: - self._contextFactory = create_instance( - objcls=self._contextFactoryClass, - settings=settings, - crawler=crawler, - method=self._sslMethod, - ) - except TypeError: - # use context factory defaults - self._contextFactory = create_instance( - objcls=self._contextFactoryClass, - settings=settings, - crawler=crawler, - ) - msg = f""" - '{settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]}' does not accept `method` \ - argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD) and/or \ - `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" - warnings.warn(msg) + self._contextFactory = load_context_factory_from_settings(settings, crawler) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') @@ -121,8 +98,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): with this endpoint comes from the pool and a CONNECT has already been issued for it. """ - - _responseMatcher = re.compile(br'HTTP/1\.. (?P\d{3})(?P.{,32})') + _truncatedLength = 1000 + _responseAnswer = r'HTTP/1\.. (?P\d{3})(?P.{,' + str(_truncatedLength) + r'})' + _responseMatcher = re.compile(_responseAnswer.encode()) def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf @@ -167,7 +145,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): extra = {'status': int(respm.group('status')), 'reason': respm.group('reason').strip()} else: - extra = rcvd_bytes[:32] + extra = rcvd_bytes[:self._truncatedLength] self._tunnelReadyDeferred.errback( TunnelError('Could not open CONNECT tunnel with proxy ' f'{self._host}:{self._port} [{extra!r}]') @@ -235,7 +213,7 @@ class TunnelingAgent(Agent): # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy - key = key + self._proxyConf + key += self._proxyConf return super()._requestWithEndpoint( key=key, endpoint=endpoint, @@ -298,16 +276,19 @@ class ScrapyAgent: bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') if proxy: - _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) + proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams if omitConnectTunnel: - warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Crawlera, it doesn't require this mode anymore, " - "so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.", - ScrapyDeprecationWarning) + warnings.warn( + "Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Zyte Smart Proxy Manager, it doesn't require " + "this mode anymore, so you should update scrapy-crawlera " + "to scrapy-zyte-smartproxy and remove '?noconnect' " + "from the Zyte Smart Proxy Manager URL.", + ScrapyDeprecationWarning, + ) if scheme == b'https' and not omitConnectTunnel: proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) @@ -320,9 +301,13 @@ class ScrapyAgent: pool=self._pool, ) else: + proxyScheme = proxyScheme or b'http' + proxyHost = to_bytes(proxyHost, encoding='ascii') + proxyPort = to_bytes(str(proxyPort), encoding='ascii') + proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) return self._ProxyAgent( reactor=reactor, - proxyURI=to_bytes(proxy, encoding='ascii'), + proxyURI=to_bytes(proxyURI, encoding='ascii'), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool, @@ -378,7 +363,38 @@ class ScrapyAgent: request.meta['download_latency'] = time() - start_time return result + @staticmethod + def _headers_from_twisted_response(response): + headers = Headers() + if response.length != UNKNOWN_LENGTH: + headers[b'Content-Length'] = str(response.length).encode() + headers.update(response.headers.getAllRawHeaders()) + return headers + def _cb_bodyready(self, txresponse, request): + headers_received_result = self._crawler.signals.send_catch_log( + signal=signals.headers_received, + headers=self._headers_from_twisted_response(txresponse), + body_length=txresponse.length, + request=request, + spider=self._crawler.spider, + ) + for handler, result in headers_received_result: + if isinstance(result, Failure) and isinstance(result.value, StopDownload): + logger.debug("Download stopped for %(request)s from signal handler %(handler)s", + {"request": request, "handler": handler.__qualname__}) + txresponse._transport.stopProducing() + with suppress(AttributeError): + txresponse._transport._producer.loseConnection() + return { + "txresponse": txresponse, + "body": b"", + "flags": ["download_stopped"], + "certificate": None, + "ip_address": None, + "failure": result if result.value.fail else None, + } + # deliverBody hangs for responses without body if txresponse.length == 0: return { @@ -432,8 +448,13 @@ class ScrapyAgent: return d def _cb_bodydone(self, result, request, url): - headers = Headers(result["txresponse"].headers.getAllRawHeaders()) + headers = self._headers_from_twisted_response(result["txresponse"]) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) + try: + version = result["txresponse"].version + protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}" + except (AttributeError, TypeError, IndexError): + protocol = None response = respcls( url=url, status=int(result["txresponse"].code), @@ -442,6 +463,7 @@ class ScrapyAgent: flags=result["flags"], certificate=result["certificate"], ip_address=result["ip_address"], + protocol=protocol, ) if result.get("failure"): result["failure"].value.response = response @@ -520,6 +542,7 @@ class _ResponseReader(protocol.Protocol): if isinstance(result, Failure) and isinstance(result.value, StopDownload): logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": self._request, "handler": handler.__qualname__}) + self.transport.stopProducing() self.transport._producer.loseConnection() failure = result if result.value.fail else None self._finish_response(flags=["download_stopped"], failure=failure) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py new file mode 100644 index 000000000..7bb88a193 --- /dev/null +++ b/scrapy/core/downloader/handlers/http2.py @@ -0,0 +1,129 @@ +import warnings +from time import time +from typing import Optional, Type, TypeVar +from urllib.parse import urldefrag + +from twisted.internet.base import DelayedCall +from twisted.internet.defer import Deferred +from twisted.internet.error import TimeoutError +from twisted.web.client import URI + +from scrapy.core.downloader.contextfactory import load_context_factory_from_settings +from scrapy.core.downloader.webclient import _parse +from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent +from scrapy.crawler import Crawler +from scrapy.http import Request, Response +from scrapy.settings import Settings +from scrapy.spiders import Spider +from scrapy.utils.python import to_bytes + + +H2DownloadHandlerOrSubclass = TypeVar("H2DownloadHandlerOrSubclass", bound="H2DownloadHandler") + + +class H2DownloadHandler: + def __init__(self, settings: Settings, crawler: Optional[Crawler] = None): + self._crawler = crawler + + from twisted.internet import reactor + self._pool = H2ConnectionPool(reactor, settings) + self._context_factory = load_context_factory_from_settings(settings, crawler) + + @classmethod + def from_crawler(cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler) -> H2DownloadHandlerOrSubclass: + return cls(crawler.settings, crawler) + + def download_request(self, request: Request, spider: Spider) -> Deferred: + agent = ScrapyH2Agent( + context_factory=self._context_factory, + pool=self._pool, + crawler=self._crawler, + ) + return agent.download_request(request, spider) + + def close(self) -> None: + self._pool.close_connections() + + +class ScrapyH2Agent: + _Agent = H2Agent + _ProxyAgent = ScrapyProxyH2Agent + + def __init__( + self, context_factory, + pool: H2ConnectionPool, + connect_timeout: int = 10, + bind_address: Optional[bytes] = None, + crawler: Optional[Crawler] = None, + ) -> None: + self._context_factory = context_factory + self._connect_timeout = connect_timeout + self._bind_address = bind_address + self._pool = pool + self._crawler = crawler + + def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: + from twisted.internet import reactor + bind_address = request.meta.get('bindaddress') or self._bind_address + proxy = request.meta.get('proxy') + if proxy: + _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) + scheme = _parse(request.url)[0] + proxy_host = proxy_host.decode() + omit_connect_tunnel = b'noconnect' in proxy_params + if omit_connect_tunnel: + warnings.warn( + "Using HTTPS proxies in the noconnect mode is not " + "supported by the downloader handler. If you use Zyte " + "Smart Proxy Manager, it doesn't require this mode " + "anymore, so you should update scrapy-crawlera to " + "scrapy-zyte-smartproxy and remove '?noconnect' from the " + "Zyte Smart Proxy Manager URL." + ) + + if scheme == b'https' and not omit_connect_tunnel: + # ToDo + raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported') + return self._ProxyAgent( + reactor=reactor, + context_factory=self._context_factory, + proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')), + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool, + ) + + return self._Agent( + reactor=reactor, + context_factory=self._context_factory, + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool, + ) + + def download_request(self, request: Request, spider: Spider) -> Deferred: + from twisted.internet import reactor + timeout = request.meta.get('download_timeout') or self._connect_timeout + agent = self._get_agent(request, timeout) + + start_time = time() + d = agent.request(request, spider) + d.addCallback(self._cb_latency, request, start_time) + + timeout_cl = reactor.callLater(timeout, d.cancel) + d.addBoth(self._cb_timeout, request, timeout, timeout_cl) + return d + + @staticmethod + def _cb_latency(response: Response, request: Request, start_time: float) -> Response: + request.meta['download_latency'] = time() - start_time + return response + + @staticmethod + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl: DelayedCall) -> Response: + if timeout_cl.active(): + timeout_cl.cancel() + return response + + url = urldefrag(request.url)[0] + raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 1966570d4..51ca1ed5e 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,5 +1,3 @@ -from urllib.parse import unquote - from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available @@ -12,6 +10,7 @@ class S3DownloadHandler: def __init__(self, settings, *, crawler=None, aws_access_key_id=None, aws_secret_access_key=None, + aws_session_token=None, httpdownloadhandler=HTTPDownloadHandler, **kw): if not is_botocore_available(): raise NotConfigured('missing botocore library') @@ -20,6 +19,8 @@ class S3DownloadHandler: aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] if not aws_secret_access_key: aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY'] + if not aws_session_token: + aws_session_token = settings['AWS_SESSION_TOKEN'] # If no credentials could be found anywhere, # consider this an anonymous connection request by default; @@ -38,7 +39,7 @@ class S3DownloadHandler: if not self.anon: SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] self._signer = SignerCls(botocore.credentials.Credentials( - aws_access_key_id, aws_secret_access_key)) + aws_access_key_id, aws_secret_access_key, aws_session_token)) _http_handler = create_instance( objcls=httpdownloadhandler, @@ -59,7 +60,7 @@ class S3DownloadHandler: url = f'{scheme}://{bucket}.s3.amazonaws.com{path}' if self.anon: request = request.replace(url=url) - elif self._signer is not None: + else: import botocore.awsrequest awsrequest = botocore.awsrequest.AWSRequest( method=request.method, @@ -69,14 +70,4 @@ class S3DownloadHandler: self._signer.add_auth(awsrequest) request = request.replace( url=url, headers=awsrequest.headers.items()) - else: - signed_headers = self.conn.make_request( - method=request.method, - bucket=bucket, - key=unquote(p.path), - query_args=unquote(p.query), - headers=request.headers, - data=request.body, - ) - request = request.replace(url=url, headers=signed_headers) return self._download_http(request, spider) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index b0e612e43..289147466 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,8 +3,12 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from twisted.internet import defer +from typing import Callable, Union, cast +from twisted.internet import defer +from twisted.python.failure import Failure + +from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager @@ -29,15 +33,15 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_exception'): self.methods['process_exception'].appendleft(mw.process_exception) - def download(self, download_func, request, spider): + def download(self, download_func: Callable, request: Request, spider: Spider): @defer.inlineCallbacks - def process_request(request): + def process_request(request: Request): for method in self.methods['process_request']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_request must return None, Response or " + f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {response.__class__.__name__}" ) if response: @@ -45,18 +49,18 @@ class DownloaderMiddlewareManager(MiddlewareManager): return (yield download_func(request=request, spider=spider)) @defer.inlineCallbacks - def process_response(response): + def process_response(response: Union[Response, Request]): if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): return response for method in self.methods['process_response']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_response must return Response or Request, " + f"Middleware {method.__qualname__} must return Response or Request, " f"got {type(response)}" ) if isinstance(response, Request): @@ -64,14 +68,14 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response @defer.inlineCallbacks - def process_exception(failure): + def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - f"Middleware {method.__self__.__class__.__name__}" - ".process_exception must return None, Response or " + f"Middleware {method.__qualname__} must return None, Response or " f"Request, got {type(response)}" ) if response: diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 2b8990b75..19a56d9b6 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -65,14 +65,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions): verifyHostname(connection, self._hostnameASCII) except (CertificateError, VerificationError) as e: logger.warning( - 'Remote certificate is not valid for hostname "{}"; {}'.format( - self._hostnameASCII, e)) + 'Remote certificate is not valid for hostname "%s"; %s', + self._hostnameASCII, e) except ValueError as e: logger.warning( 'Ignoring error while verifying certificate ' - 'from host "{}" (exception: {})'.format( - self._hostnameASCII, repr(e))) + 'from host "%s" (exception: %r)', + self._hostnameASCII, e) DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index c13683393..06cb96489 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,13 +1,14 @@ +import re from time import time from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.http import HTTPClient -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.protocol import ClientFactory from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode from scrapy.responsetypes import responsetypes @@ -32,6 +33,8 @@ def _parse(url): and is ascii-only. """ url = url.strip() + if not re.match(r'^\w+://', url): + url = '//' + url parsed = urlparse(url) return _parsed_url_args(parsed) @@ -110,7 +113,7 @@ class ScrapyHTTPClientFactory(ClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - return respcls(url=self._url, status=status, headers=headers, body=body) + return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version)) def _set_connection_attributes(self, request): parsed = urlparse_cached(request) @@ -167,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory): p.followRedirect = self.followRedirect p.afterFoundGet = self.afterFoundGet if self.timeout: + from twisted.internet import reactor timeoutCall = reactor.callLater(self.timeout, p.timeout) self.deferred.addBoth(self._cancelTimeout, timeoutCall) return p diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 93bcdb49a..f9de7ee23 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -1,51 +1,66 @@ """ -This is the Scrapy engine which controls the Scheduler, Downloader and Spiders. +This is the Scrapy engine which controls the Scheduler, Downloader and Spider. For more information see docs/topics/architecture.rst """ import logging +import warnings from time import time +from typing import Callable, Iterable, Iterator, Optional, Set, Union -from twisted.internet import defer, task +from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.task import LoopingCall from twisted.python.failure import Failure from scrapy import signals from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider +from scrapy.exceptions import ( + CloseSpider, + DontCloseSpider, + ScrapyDeprecationWarning, +) from scrapy.http import Response, Request -from scrapy.utils.misc import load_object -from scrapy.utils.reactor import CallLaterOnce +from scrapy.settings import BaseSettings +from scrapy.spiders import Spider from scrapy.utils.log import logformatter_adapter, failure_to_exc_info +from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.reactor import CallLaterOnce + logger = logging.getLogger(__name__) class Slot: - - def __init__(self, start_requests, close_if_idle, nextcall, scheduler): - self.closing = False - self.inprogress = set() # requests in progress - self.start_requests = iter(start_requests) + def __init__( + self, + start_requests: Iterable, + close_if_idle: bool, + nextcall: CallLaterOnce, + scheduler, + ) -> None: + self.closing: Optional[Deferred] = None + self.inprogress: Set[Request] = set() + self.start_requests: Optional[Iterator] = iter(start_requests) self.close_if_idle = close_if_idle self.nextcall = nextcall self.scheduler = scheduler - self.heartbeat = task.LoopingCall(nextcall.schedule) + self.heartbeat = LoopingCall(nextcall.schedule) - def add_request(self, request): + def add_request(self, request: Request) -> None: self.inprogress.add(request) - def remove_request(self, request): + def remove_request(self, request: Request) -> None: self.inprogress.remove(request) self._maybe_fire_closing() - def close(self): - self.closing = defer.Deferred() + def close(self) -> Deferred: + self.closing = Deferred() self._maybe_fire_closing() return self.closing - def _maybe_fire_closing(self): - if self.closing and not self.inprogress: + def _maybe_fire_closing(self) -> None: + if self.closing is not None and not self.inprogress: if self.nextcall: self.nextcall.cancel() if self.heartbeat.running: @@ -54,210 +69,236 @@ class Slot: class ExecutionEngine: - - def __init__(self, crawler, spider_closed_callback): + def __init__(self, crawler, spider_closed_callback: Callable) -> None: self.crawler = crawler self.settings = crawler.settings self.signals = crawler.signals self.logformatter = crawler.logformatter - self.slot = None - self.spider = None + self.slot: Optional[Slot] = None + self.spider: Optional[Spider] = None self.running = False self.paused = False - self.scheduler_cls = load_object(self.settings['SCHEDULER']) + self.scheduler_cls = self._get_scheduler_class(crawler.settings) downloader_cls = load_object(self.settings['DOWNLOADER']) self.downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) self._spider_closed_callback = spider_closed_callback - @defer.inlineCallbacks - def start(self): - """Start the execution engine""" + def _get_scheduler_class(self, settings: BaseSettings) -> type: + from scrapy.core.scheduler import BaseScheduler + scheduler_cls = load_object(settings["SCHEDULER"]) + if not issubclass(scheduler_cls, BaseScheduler): + raise TypeError( + f"The provided scheduler class ({settings['SCHEDULER']})" + " does not fully implement the scheduler interface" + ) + return scheduler_cls + + @inlineCallbacks + def start(self) -> Deferred: if self.running: raise RuntimeError("Engine already running") self.start_time = time() yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True - self._closewait = defer.Deferred() + self._closewait = Deferred() yield self._closewait - def stop(self): - """Stop the execution engine gracefully""" + def stop(self) -> Deferred: + """Gracefully stop the execution engine""" + @inlineCallbacks + def _finish_stopping_engine(_) -> Deferred: + yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + self._closewait.callback(None) + if not self.running: raise RuntimeError("Engine not running") + self.running = False - dfd = self._close_all_spiders() - return dfd.addBoth(lambda _: self._finish_stopping_engine()) + dfd = self.close_spider(self.spider, reason="shutdown") if self.spider is not None else succeed(None) + return dfd.addBoth(_finish_stopping_engine) - def close(self): - """Close the execution engine gracefully. - - If it has already been started, stop it. In all cases, close all spiders - and the downloader. + def close(self) -> Deferred: + """ + Gracefully close the execution engine. + If it has already been started, stop it. In all cases, close the spider and the downloader. """ if self.running: - # Will also close spiders and downloader - return self.stop() - elif self.open_spiders: - # Will also close downloader - return self._close_all_spiders() - else: - return defer.succeed(self.downloader.close()) + return self.stop() # will also close spider and downloader + if self.spider is not None: + return self.close_spider(self.spider, reason="shutdown") # will also close downloader + return succeed(self.downloader.close()) - def pause(self): - """Pause the execution engine""" + def pause(self) -> None: self.paused = True - def unpause(self): - """Resume the execution engine""" + def unpause(self) -> None: self.paused = False - def _next_request(self, spider): - slot = self.slot - if not slot: - return + def _next_request(self) -> None: + assert self.slot is not None # typing + assert self.spider is not None # typing if self.paused: - return + return None - while not self._needs_backout(spider): - if not self._next_request_from_scheduler(spider): - break + while not self._needs_backout() and self._next_request_from_scheduler() is not None: + pass - if slot.start_requests and not self._needs_backout(spider): + if self.slot.start_requests is not None and not self._needs_backout(): try: - request = next(slot.start_requests) + request = next(self.slot.start_requests) except StopIteration: - slot.start_requests = None + self.slot.start_requests = None except Exception: - slot.start_requests = None - logger.error('Error while obtaining start requests', - exc_info=True, extra={'spider': spider}) + self.slot.start_requests = None + logger.error('Error while obtaining start requests', exc_info=True, extra={'spider': self.spider}) else: - self.crawl(request, spider) + self.crawl(request) - if self.spider_is_idle(spider) and slot.close_if_idle: - self._spider_idle(spider) + if self.spider_is_idle() and self.slot.close_if_idle: + self._spider_idle() - def _needs_backout(self, spider): - slot = self.slot + def _needs_backout(self) -> bool: return ( not self.running - or slot.closing + or self.slot.closing # type: ignore[union-attr] or self.downloader.needs_backout() - or self.scraper.slot.needs_backout() + or self.scraper.slot.needs_backout() # type: ignore[union-attr] ) - def _next_request_from_scheduler(self, spider): - slot = self.slot - request = slot.scheduler.next_request() - if not request: - return - d = self._download(request, spider) - d.addBoth(self._handle_downloader_output, request, spider) + def _next_request_from_scheduler(self) -> Optional[Deferred]: + assert self.slot is not None # typing + assert self.spider is not None # typing + + request = self.slot.scheduler.next_request() + if request is None: + return None + + d = self._download(request, self.spider) + d.addBoth(self._handle_downloader_output, request) d.addErrback(lambda f: logger.info('Error while handling downloader output', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) - d.addBoth(lambda _: slot.remove_request(request)) + extra={'spider': self.spider})) + d.addBoth(lambda _: self.slot.remove_request(request)) d.addErrback(lambda f: logger.info('Error while removing request from slot', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) - d.addBoth(lambda _: slot.nextcall.schedule()) + extra={'spider': self.spider})) + d.addBoth(lambda _: self.slot.nextcall.schedule()) d.addErrback(lambda f: logger.info('Error while scheduling new request', exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + extra={'spider': self.spider})) return d - def _handle_downloader_output(self, response, request, spider): - if not isinstance(response, (Request, Response, Failure)): - raise TypeError( - "Incorrect type: expected Request, Response or Failure, got " - f"{type(response)}: {response!r}" - ) + def _handle_downloader_output( + self, result: Union[Request, Response, Failure], request: Request + ) -> Optional[Deferred]: + assert self.spider is not None # typing + + if not isinstance(result, (Request, Response, Failure)): + raise TypeError(f"Incorrect type: expected Request, Response or Failure, got {type(result)}: {result!r}") + # downloader middleware can return requests (for example, redirects) - if isinstance(response, Request): - self.crawl(response, spider) - return - # response is a Response or Failure - d = self.scraper.enqueue_scrape(response, request, spider) - d.addErrback(lambda f: logger.error('Error while enqueuing downloader output', - exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + if isinstance(result, Request): + self.crawl(result) + return None + + d = self.scraper.enqueue_scrape(result, request, self.spider) + d.addErrback( + lambda f: logger.error( + "Error while enqueuing downloader output", + exc_info=failure_to_exc_info(f), + extra={'spider': self.spider}, + ) + ) return d - def spider_is_idle(self, spider): - if not self.scraper.slot.is_idle(): - # scraper is not idle + def spider_is_idle(self, spider: Optional[Spider] = None) -> bool: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: + raise RuntimeError("Engine slot not assigned") + if not self.scraper.slot.is_idle(): # type: ignore[union-attr] return False - - if self.downloader.active: - # downloader has pending requests + if self.downloader.active: # downloader has pending requests return False - - if self.slot.start_requests is not None: - # not all start requests are handled + if self.slot.start_requests is not None: # not all start requests are handled return False - if self.slot.scheduler.has_pending_requests(): - # scheduler has pending requests return False - return True - @property - def open_spiders(self): - return [self.spider] if self.spider else [] + def crawl(self, request: Request, spider: Optional[Spider] = None) -> None: + """Inject the request into the spider <-> downloader pipeline""" + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if spider is not self.spider: + raise RuntimeError(f"The spider {spider.name!r} does not match the open spider") + if self.spider is None: + raise RuntimeError(f"No open spider to crawl: {request}") + self._schedule_request(request, self.spider) + self.slot.nextcall.schedule() # type: ignore[union-attr] - def has_capacity(self): - """Does the engine have capacity to handle more spiders""" - return not bool(self.slot) - - def crawl(self, request, spider): - if spider not in self.open_spiders: - raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}") - self.schedule(request, spider) - self.slot.nextcall.schedule() - - def schedule(self, request, spider): + def _schedule_request(self, request: Request, spider: Spider) -> None: self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider) - if not self.slot.scheduler.enqueue_request(request): + if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider) - def download(self, request, spider): - d = self._download(request, spider) - d.addBoth(self._downloaded, self.slot, request, spider) - return d + def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: + """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" + if spider is None: + spider = self.spider + else: + warnings.warn( + "Passing a 'spider' argument to ExecutionEngine.download is deprecated", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if spider is not self.spider: + logger.warning("The spider '%s' does not match the open spider", spider.name) + if spider is None: + raise RuntimeError(f"No open spider to crawl: {request}") + return self._download(request, spider).addBoth(self._downloaded, request, spider) - def _downloaded(self, response, slot, request, spider): - slot.remove_request(request) - return self.download(response, spider) if isinstance(response, Request) else response + def _downloaded( + self, result: Union[Response, Request], request: Request, spider: Spider + ) -> Union[Deferred, Response]: + assert self.slot is not None # typing + self.slot.remove_request(request) + return self.download(result, spider) if isinstance(result, Request) else result - def _download(self, request, spider): - slot = self.slot - slot.add_request(request) + def _download(self, request: Request, spider: Spider) -> Deferred: + assert self.slot is not None # typing - def _on_success(response): - if not isinstance(response, (Response, Request)): - raise TypeError( - "Incorrect type: expected Response or Request, got " - f"{type(response)}: {response!r}" - ) - if isinstance(response, Response): - if response.request is None: - response.request = request - logkws = self.logformatter.crawled(response.request, response, spider) + self.slot.add_request(request) + + def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: + if not isinstance(result, (Response, Request)): + raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}") + if isinstance(result, Response): + if result.request is None: + result.request = request + logkws = self.logformatter.crawled(result.request, result, spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) self.signals.send_catch_log( signal=signals.response_received, - response=response, - request=response.request, + response=result, + request=result.request, spider=spider, ) - return response + return result def _on_complete(_): - slot.nextcall.schedule() + self.slot.nextcall.schedule() return _ dwld = self.downloader.fetch(request, spider) @@ -265,58 +306,62 @@ class ExecutionEngine: dwld.addBoth(_on_complete) return dwld - @defer.inlineCallbacks - def open_spider(self, spider, start_requests=(), close_if_idle=True): - if not self.has_capacity(): + @inlineCallbacks + def open_spider(self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True): + if self.slot is not None: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={'spider': spider}) - nextcall = CallLaterOnce(self._next_request, spider) - scheduler = self.scheduler_cls.from_crawler(self.crawler) + nextcall = CallLaterOnce(self._next_request) + scheduler = create_instance(self.scheduler_cls, settings=None, crawler=self.crawler) start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider) - slot = Slot(start_requests, close_if_idle, nextcall, scheduler) - self.slot = slot + self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler) self.spider = spider - yield scheduler.open(spider) + if hasattr(scheduler, "open"): + yield scheduler.open(spider) yield self.scraper.open_spider(spider) self.crawler.stats.open_spider(spider) yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) - slot.nextcall.schedule() - slot.heartbeat.start(5) + self.slot.nextcall.schedule() + self.slot.heartbeat.start(5) - def _spider_idle(self, spider): - """Called when a spider gets idle. This function is called when there - are no remaining pages to download or schedule. It can be called - multiple times. If some extension raises a DontCloseSpider exception - (in the spider_idle signal handler) the spider is not closed until the - next loop and this function is guaranteed to be called (at least) once - again for this spider. + def _spider_idle(self) -> None: """ - res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider) - if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): - return + Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. + It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider + exception, the spider is not closed until the next loop and this function is guaranteed to be called + (at least) once again. A handler can raise CloseSpider to provide a custom closing reason. + """ + assert self.spider is not None # typing + expected_ex = (DontCloseSpider, CloseSpider) + res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex) + detected_ex = { + ex: x.value + for _, x in res + for ex in expected_ex + if isinstance(x, Failure) and isinstance(x.value, ex) + } + if DontCloseSpider in detected_ex: + return None + if self.spider_is_idle(): + ex = detected_ex.get(CloseSpider, CloseSpider(reason='finished')) + assert isinstance(ex, CloseSpider) # typing + self.close_spider(self.spider, reason=ex.reason) - if self.spider_is_idle(spider): - self.close_spider(spider, reason='finished') - - def close_spider(self, spider, reason='cancelled'): + def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: """Close (cancel) spider and clear all its outstanding requests""" + if self.slot is None: + raise RuntimeError("Engine slot not assigned") - slot = self.slot - if slot.closing: - return slot.closing - logger.info("Closing spider (%(reason)s)", - {'reason': reason}, - extra={'spider': spider}) + if self.slot.closing is not None: + return self.slot.closing - dfd = slot.close() + logger.info("Closing spider (%(reason)s)", {'reason': reason}, extra={'spider': spider}) - def log_failure(msg): - def errback(failure): - logger.error( - msg, - exc_info=failure_to_exc_info(failure), - extra={'spider': spider} - ) + dfd = self.slot.close() + + def log_failure(msg: str) -> Callable: + def errback(failure: Failure) -> None: + logger.error(msg, exc_info=failure_to_exc_info(failure), extra={'spider': spider}) return errback dfd.addBoth(lambda _: self.downloader.close()) @@ -325,19 +370,19 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.scraper.close_spider(spider)) dfd.addErrback(log_failure('Scraper close failure')) - dfd.addBoth(lambda _: slot.scheduler.close(reason)) - dfd.addErrback(log_failure('Scheduler close failure')) + if hasattr(self.slot.scheduler, "close"): + dfd.addBoth(lambda _: self.slot.scheduler.close(reason)) + dfd.addErrback(log_failure("Scheduler close failure")) dfd.addBoth(lambda _: self.signals.send_catch_log_deferred( - signal=signals.spider_closed, spider=spider, reason=reason)) + signal=signals.spider_closed, spider=spider, reason=reason, + )) dfd.addErrback(log_failure('Error while sending spider_close signal')) dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason)) dfd.addErrback(log_failure('Stats close failure')) - dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", - {'reason': reason}, - extra={'spider': spider})) + dfd.addBoth(lambda _: logger.info("Spider closed (%(reason)s)", {'reason': reason}, extra={'spider': spider})) dfd.addBoth(lambda _: setattr(self, 'slot', None)) dfd.addErrback(log_failure('Error while unassigning slot')) @@ -349,12 +394,26 @@ class ExecutionEngine: return dfd - def _close_all_spiders(self): - dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders] - dlist = defer.DeferredList(dfds) - return dlist + @property + def open_spiders(self) -> list: + warnings.warn( + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return [self.spider] if self.spider is not None else [] - @defer.inlineCallbacks - def _finish_stopping_engine(self): - yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) - self._closewait.callback(None) + def has_capacity(self) -> bool: + warnings.warn("ExecutionEngine.has_capacity is deprecated", ScrapyDeprecationWarning, stacklevel=2) + return not bool(self.slot) + + def schedule(self, request: Request, spider: Spider) -> None: + warnings.warn( + "ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: + raise RuntimeError("Engine slot not assigned") + self._schedule_request(request, spider) diff --git a/scrapy/core/http2/__init__.py b/scrapy/core/http2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py new file mode 100644 index 000000000..f7b0c3f99 --- /dev/null +++ b/scrapy/core/http2/agent.py @@ -0,0 +1,157 @@ +from collections import deque +from typing import Deque, Dict, List, Optional, Tuple + +from twisted.internet import defer +from twisted.internet.base import ReactorBase +from twisted.internet.defer import Deferred +from twisted.internet.endpoints import HostnameEndpoint +from twisted.python.failure import Failure +from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory +from twisted.web.error import SchemeNotSupported + +from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory +from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory +from scrapy.http.request import Request +from scrapy.settings import Settings +from scrapy.spiders import Spider + + +class H2ConnectionPool: + def __init__(self, reactor: ReactorBase, settings: Settings) -> None: + self._reactor = reactor + self.settings = settings + + # Store a dictionary which is used to get the respective + # H2ClientProtocolInstance using the key as Tuple(scheme, hostname, port) + self._connections: Dict[Tuple, H2ClientProtocol] = {} + + # Save all requests that arrive before the connection is established + self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} + + def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: + if key in self._pending_requests: + # Received a request while connecting to remote + # Create a deferred which will fire with the H2ClientProtocol + # instance + d = Deferred() + self._pending_requests[key].append(d) + return d + + # Check if we already have a connection to the remote + conn = self._connections.get(key, None) + if conn: + # Return this connection instance wrapped inside a deferred + return defer.succeed(conn) + + # No connection is established for the given URI + return self._new_connection(key, uri, endpoint) + + def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: + self._pending_requests[key] = deque() + + conn_lost_deferred = Deferred() + conn_lost_deferred.addCallback(self._remove_connection, key) + + factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) + conn_d = endpoint.connect(factory) + conn_d.addCallback(self.put_connection, key) + + d = Deferred() + self._pending_requests[key].append(d) + return d + + def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol: + self._connections[key] = conn + + # Now as we have established a proper HTTP/2 connection + # we fire all the deferred's with the connection instance + pending_requests = self._pending_requests.pop(key, None) + while pending_requests: + d = pending_requests.popleft() + d.callback(conn) + + return conn + + def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None: + self._connections.pop(key) + + # Call the errback of all the pending requests for this connection + pending_requests = self._pending_requests.pop(key, None) + while pending_requests: + d = pending_requests.popleft() + d.errback(errors) + + def close_connections(self) -> None: + """Close all the HTTP/2 connections and remove them from pool + + Returns: + Deferred that fires when all connections have been closed + """ + for conn in self._connections.values(): + conn.transport.abortConnection() + + +class H2Agent: + def __init__( + self, + reactor: ReactorBase, + pool: H2ConnectionPool, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, + bind_address: Optional[bytes] = None, + ) -> None: + self._reactor = reactor + self._pool = pool + self._context_factory = AcceptableProtocolsContextFactory(context_factory, acceptable_protocols=[b'h2']) + self.endpoint_factory = _StandardEndpointFactory( + self._reactor, self._context_factory, connect_timeout, bind_address + ) + + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(uri) + + def get_key(self, uri: URI) -> Tuple: + """ + Arguments: + uri - URI obtained directly from request URL + """ + return uri.scheme, uri.host, uri.port + + def request(self, request: Request, spider: Spider) -> Deferred: + uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) + try: + endpoint = self.get_endpoint(uri) + except SchemeNotSupported: + return defer.fail(Failure()) + + key = self.get_key(uri) + d = self._pool.get_connection(key, uri, endpoint) + d.addCallback(lambda conn: conn.request(request, spider)) + return d + + +class ScrapyProxyH2Agent(H2Agent): + def __init__( + self, + reactor: ReactorBase, + proxy_uri: URI, + pool: H2ConnectionPool, + context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, + bind_address: Optional[bytes] = None, + ) -> None: + super(ScrapyProxyH2Agent, self).__init__( + reactor=reactor, + pool=pool, + context_factory=context_factory, + connect_timeout=connect_timeout, + bind_address=bind_address, + ) + self._proxy_uri = proxy_uri + + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(self._proxy_uri) + + def get_key(self, uri: URI) -> Tuple: + """We use the proxy uri instead of uri obtained from request url""" + return "http-proxy", self._proxy_uri.host, self._proxy_uri.port diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py new file mode 100644 index 000000000..1d150b7ce --- /dev/null +++ b/scrapy/core/http2/protocol.py @@ -0,0 +1,418 @@ +import ipaddress +import itertools +import logging +from collections import deque +from ipaddress import IPv4Address, IPv6Address +from typing import Dict, List, Optional, Union + +from h2.config import H2Configuration +from h2.connection import H2Connection +from h2.errors import ErrorCodes +from h2.events import ( + Event, ConnectionTerminated, DataReceived, ResponseReceived, + SettingsAcknowledged, StreamEnded, StreamReset, UnknownFrameReceived, + WindowUpdated +) +from h2.exceptions import FrameTooLargeError, H2Error +from twisted.internet.defer import Deferred +from twisted.internet.error import TimeoutError +from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory +from twisted.internet.protocol import connectionDone, Factory, Protocol +from twisted.internet.ssl import Certificate +from twisted.protocols.policies import TimeoutMixin +from twisted.python.failure import Failure +from twisted.web.client import URI +from zope.interface import implementer + +from scrapy.core.http2.stream import Stream, StreamCloseReason +from scrapy.http import Request +from scrapy.settings import Settings +from scrapy.spiders import Spider + + +logger = logging.getLogger(__name__) + + +PROTOCOL_NAME = b"h2" + + +class InvalidNegotiatedProtocol(H2Error): + + def __init__(self, negotiated_protocol: bytes) -> None: + self.negotiated_protocol = negotiated_protocol + + def __str__(self) -> str: + return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}") + + +class RemoteTerminatedConnection(H2Error): + def __init__( + self, + remote_ip_address: Optional[Union[IPv4Address, IPv6Address]], + event: ConnectionTerminated, + ) -> None: + self.remote_ip_address = remote_ip_address + self.terminate_event = event + + def __str__(self) -> str: + return f'Received GOAWAY frame from {self.remote_ip_address!r}' + + +class MethodNotAllowed405(H2Error): + def __init__(self, remote_ip_address: Optional[Union[IPv4Address, IPv6Address]]) -> None: + self.remote_ip_address = remote_ip_address + + def __str__(self) -> str: + return f"Received 'HTTP/2.0 405 Method Not Allowed' from {self.remote_ip_address!r}" + + +@implementer(IHandshakeListener) +class H2ClientProtocol(Protocol, TimeoutMixin): + IDLE_TIMEOUT = 240 + + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None: + """ + Arguments: + uri -- URI of the base url to which HTTP/2 Connection will be made. + uri is used to verify that incoming client requests have correct + base URL. + settings -- Scrapy project settings + conn_lost_deferred -- Deferred fires with the reason: Failure to notify + that connection was lost + """ + self._conn_lost_deferred = conn_lost_deferred + + config = H2Configuration(client_side=True, header_encoding='utf-8') + self.conn = H2Connection(config=config) + + # ID of the next request stream + # Following the convention - 'Streams initiated by a client MUST + # use odd-numbered stream identifiers' (RFC 7540 - Section 5.1.1) + self._stream_id_generator = itertools.count(start=1, step=2) + + # Streams are stored in a dictionary keyed off their stream IDs + self.streams: Dict[int, Stream] = {} + + # If requests are received before connection is made we keep + # all requests in a pool and send them as the connection is made + self._pending_request_stream_pool: deque = deque() + + # Save an instance of errors raised which lead to losing the connection + # We pass these instances to the streams ResponseFailed() failure + self._conn_lost_errors: List[BaseException] = [] + + # Some meta data of this connection + # initialized when connection is successfully made + self.metadata: Dict = { + # Peer certificate instance + 'certificate': None, + + # Address of the server we are connected to which + # is updated when HTTP/2 connection is made successfully + 'ip_address': None, + + # URI of the peer HTTP/2 connection is made + 'uri': uri, + + # Both ip_address and uri are used by the Stream before + # initiating the request to verify that the base address + + # Variables taken from Project Settings + 'default_download_maxsize': settings.getint('DOWNLOAD_MAXSIZE'), + 'default_download_warnsize': settings.getint('DOWNLOAD_WARNSIZE'), + + # Counter to keep track of opened streams. This counter + # is used to make sure that not more than MAX_CONCURRENT_STREAMS + # streams are opened which leads to ProtocolError + # We use simple FIFO policy to handle pending requests + 'active_streams': 0, + + # Flag to keep track if settings were acknowledged by the remote + # This ensures that we have established a HTTP/2 connection + 'settings_acknowledged': False, + } + + @property + def h2_connected(self) -> bool: + """Boolean to keep track of the connection status. + This is used while initiating pending streams to make sure + that we initiate stream only during active HTTP/2 Connection + """ + return bool(self.transport.connected) and self.metadata['settings_acknowledged'] + + @property + def allowed_max_concurrent_streams(self) -> int: + """We keep total two streams for client (sending data) and + server side (receiving data) for a single request. To be safe + we choose the minimum. Since this value can change in event + RemoteSettingsChanged we make variable a property. + """ + return min( + self.conn.local_settings.max_concurrent_streams, + self.conn.remote_settings.max_concurrent_streams + ) + + def _send_pending_requests(self) -> None: + """Initiate all pending requests from the deque following FIFO + We make sure that at any time {allowed_max_concurrent_streams} + streams are active. + """ + while ( + self._pending_request_stream_pool + and self.metadata['active_streams'] < self.allowed_max_concurrent_streams + and self.h2_connected + ): + self.metadata['active_streams'] += 1 + stream = self._pending_request_stream_pool.popleft() + stream.initiate_request() + self._write_to_transport() + + def pop_stream(self, stream_id: int) -> Stream: + """Perform cleanup when a stream is closed + """ + stream = self.streams.pop(stream_id) + self.metadata['active_streams'] -= 1 + self._send_pending_requests() + return stream + + def _new_stream(self, request: Request, spider: Spider) -> Stream: + """Instantiates a new Stream object + """ + stream = Stream( + stream_id=next(self._stream_id_generator), + request=request, + protocol=self, + download_maxsize=getattr(spider, 'download_maxsize', self.metadata['default_download_maxsize']), + download_warnsize=getattr(spider, 'download_warnsize', self.metadata['default_download_warnsize']), + ) + self.streams[stream.stream_id] = stream + return stream + + def _write_to_transport(self) -> None: + """ Write data to the underlying transport connection + from the HTTP2 connection instance if any + """ + # Reset the idle timeout as connection is still actively sending data + self.resetTimeout() + + data = self.conn.data_to_send() + self.transport.write(data) + + def request(self, request: Request, spider: Spider) -> Deferred: + if not isinstance(request, Request): + raise TypeError(f'Expected scrapy.http.Request, received {request.__class__.__qualname__}') + + stream = self._new_stream(request, spider) + d = stream.get_response() + + # Add the stream to the request pool + self._pending_request_stream_pool.append(stream) + + # If we receive a request when connection is idle + # We need to initiate pending requests + self._send_pending_requests() + return d + + def connectionMade(self) -> None: + """Called by Twisted when the connection is established. We can start + sending some data now: we should open with the connection preamble. + """ + # Initialize the timeout + self.setTimeout(self.IDLE_TIMEOUT) + + destination = self.transport.getPeer() + self.metadata['ip_address'] = ipaddress.ip_address(destination.host) + + # Initiate H2 Connection + self.conn.initiate_connection() + self._write_to_transport() + + def _lose_connection_with_error(self, errors: List[BaseException]) -> None: + """Helper function to lose the connection with the error sent as a + reason""" + self._conn_lost_errors += errors + self.transport.loseConnection() + + def handshakeCompleted(self) -> None: + """ + Close the connection if it's not made via the expected protocol + """ + if self.transport.negotiatedProtocol is not None and self.transport.negotiatedProtocol != PROTOCOL_NAME: + # we have not initiated the connection yet, no need to send a GOAWAY frame to the remote peer + self._lose_connection_with_error([InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)]) + + def _check_received_data(self, data: bytes) -> None: + """Checks for edge cases where the connection to remote fails + without raising an appropriate H2Error + + Arguments: + data -- Data received from the remote + """ + if data.startswith(b'HTTP/2.0 405 Method Not Allowed'): + raise MethodNotAllowed405(self.metadata['ip_address']) + + def dataReceived(self, data: bytes) -> None: + # Reset the idle timeout as connection is still actively receiving data + self.resetTimeout() + + try: + self._check_received_data(data) + events = self.conn.receive_data(data) + self._handle_events(events) + except H2Error as e: + if isinstance(e, FrameTooLargeError): + # hyper-h2 does not drop the connection in this scenario, we + # need to abort the connection manually. + self._conn_lost_errors += [e] + self.transport.abortConnection() + return + + # Save this error as ultimately the connection will be dropped + # internally by hyper-h2. Saved error will be passed to all the streams + # closed with the connection. + self._lose_connection_with_error([e]) + finally: + self._write_to_transport() + + def timeoutConnection(self) -> None: + """Called when the connection times out. + We lose the connection with TimeoutError""" + + # Check whether there are open streams. If there are, we're going to + # want to use the error code PROTOCOL_ERROR. If there aren't, use + # NO_ERROR. + if ( + self.conn.open_outbound_streams > 0 + or self.conn.open_inbound_streams > 0 + or self.metadata['active_streams'] > 0 + ): + error_code = ErrorCodes.PROTOCOL_ERROR + else: + error_code = ErrorCodes.NO_ERROR + self.conn.close_connection(error_code=error_code) + self._write_to_transport() + + self._lose_connection_with_error([ + TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s") + ]) + + def connectionLost(self, reason: Failure = connectionDone) -> None: + """Called by Twisted when the transport connection is lost. + No need to write anything to transport here. + """ + # Cancel the timeout if not done yet + self.setTimeout(None) + + # Notify the connection pool instance such that no new requests are + # sent over current connection + if not reason.check(connectionDone): + self._conn_lost_errors.append(reason) + + self._conn_lost_deferred.callback(self._conn_lost_errors) + + for stream in self.streams.values(): + if stream.metadata['request_sent']: + close_reason = StreamCloseReason.CONNECTION_LOST + else: + close_reason = StreamCloseReason.INACTIVE + stream.close(close_reason, self._conn_lost_errors, from_protocol=True) + + self.metadata['active_streams'] -= len(self.streams) + self.streams.clear() + self._pending_request_stream_pool.clear() + self.conn.close_connection() + + def _handle_events(self, events: List[Event]) -> None: + """Private method which acts as a bridge between the events + received from the HTTP/2 data and IH2EventsHandler + + Arguments: + events -- A list of events that the remote peer triggered by sending data + """ + for event in events: + if isinstance(event, ConnectionTerminated): + self.connection_terminated(event) + elif isinstance(event, DataReceived): + self.data_received(event) + elif isinstance(event, ResponseReceived): + self.response_received(event) + elif isinstance(event, StreamEnded): + self.stream_ended(event) + elif isinstance(event, StreamReset): + self.stream_reset(event) + elif isinstance(event, WindowUpdated): + self.window_updated(event) + elif isinstance(event, SettingsAcknowledged): + self.settings_acknowledged(event) + elif isinstance(event, UnknownFrameReceived): + logger.warning('Unknown frame received: %s', event.frame) + + # Event handler functions starts here + def connection_terminated(self, event: ConnectionTerminated) -> None: + self._lose_connection_with_error([ + RemoteTerminatedConnection(self.metadata['ip_address'], event) + ]) + + def data_received(self, event: DataReceived) -> None: + try: + stream = self.streams[event.stream_id] + except KeyError: + pass # We ignore server-initiated events + else: + stream.receive_data(event.data, event.flow_controlled_length) + + def response_received(self, event: ResponseReceived) -> None: + try: + stream = self.streams[event.stream_id] + except KeyError: + pass # We ignore server-initiated events + else: + stream.receive_headers(event.headers) + + def settings_acknowledged(self, event: SettingsAcknowledged) -> None: + self.metadata['settings_acknowledged'] = True + + # Send off all the pending requests as now we have + # established a proper HTTP/2 connection + self._send_pending_requests() + + # Update certificate when our HTTP/2 connection is established + self.metadata['certificate'] = Certificate(self.transport.getPeerCertificate()) + + def stream_ended(self, event: StreamEnded) -> None: + try: + stream = self.pop_stream(event.stream_id) + except KeyError: + pass # We ignore server-initiated events + else: + stream.close(StreamCloseReason.ENDED, from_protocol=True) + + def stream_reset(self, event: StreamReset) -> None: + try: + stream = self.pop_stream(event.stream_id) + except KeyError: + pass # We ignore server-initiated events + else: + stream.close(StreamCloseReason.RESET, from_protocol=True) + + def window_updated(self, event: WindowUpdated) -> None: + if event.stream_id != 0: + self.streams[event.stream_id].receive_window_update() + else: + # Send leftover data for all the streams + for stream in self.streams.values(): + stream.receive_window_update() + + +@implementer(IProtocolNegotiationFactory) +class H2ClientFactory(Factory): + def __init__(self, uri: URI, settings: Settings, conn_lost_deferred: Deferred) -> None: + self.uri = uri + self.settings = settings + self.conn_lost_deferred = conn_lost_deferred + + def buildProtocol(self, addr) -> H2ClientProtocol: + return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) + + def acceptableProtocols(self) -> List[bytes]: + return [PROTOCOL_NAME] diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py new file mode 100644 index 000000000..5c393c027 --- /dev/null +++ b/scrapy/core/http2/stream.py @@ -0,0 +1,470 @@ +import logging +from enum import Enum +from io import BytesIO +from urllib.parse import urlparse +from typing import Dict, List, Optional, Tuple, TYPE_CHECKING + +from h2.errors import ErrorCodes +from h2.exceptions import H2Error, ProtocolError, StreamClosedError +from hpack import HeaderTuple +from twisted.internet.defer import Deferred, CancelledError +from twisted.internet.error import ConnectionClosed +from twisted.python.failure import Failure +from twisted.web.client import ResponseFailed + +from scrapy.http import Request +from scrapy.http.headers import Headers +from scrapy.responsetypes import responsetypes + +if TYPE_CHECKING: + from scrapy.core.http2.protocol import H2ClientProtocol + + +logger = logging.getLogger(__name__) + + +class InactiveStreamClosed(ConnectionClosed): + """Connection was closed without sending request headers + of the stream. This happens when a stream is waiting for other + streams to close and connection is lost.""" + + def __init__(self, request: Request) -> None: + self.request = request + + def __str__(self) -> str: + return f'InactiveStreamClosed: Connection was closed without sending the request {self.request!r}' + + +class InvalidHostname(H2Error): + + def __init__(self, request: Request, expected_hostname: str, expected_netloc: str) -> None: + self.request = request + self.expected_hostname = expected_hostname + self.expected_netloc = expected_netloc + + def __str__(self) -> str: + return f'InvalidHostname: Expected {self.expected_hostname} or {self.expected_netloc} in {self.request}' + + +class StreamCloseReason(Enum): + # Received a StreamEnded event from the remote + ENDED = 1 + + # Received a StreamReset event -- ended abruptly + RESET = 2 + + # Transport connection was lost + CONNECTION_LOST = 3 + + # Expected response body size is more than allowed limit + MAXSIZE_EXCEEDED = 4 + + # Response deferred is cancelled by the client + # (happens when client called response_deferred.cancel()) + CANCELLED = 5 + + # Connection lost and the stream was not initiated + INACTIVE = 6 + + # The hostname of the request is not same as of connected peer hostname + # As a result sending this request will the end the connection + INVALID_HOSTNAME = 7 + + +class Stream: + """Represents a single HTTP/2 Stream. + + Stream is a bidirectional flow of bytes within an established connection, + which may carry one or more messages. Handles the transfer of HTTP Headers + and Data frames. + + Role of this class is to + 1. Combine all the data frames + """ + + def __init__( + self, + stream_id: int, + request: Request, + protocol: "H2ClientProtocol", + download_maxsize: int = 0, + download_warnsize: int = 0, + ) -> None: + """ + Arguments: + stream_id -- Unique identifier for the stream within a single HTTP/2 connection + request -- The HTTP request associated to the stream + protocol -- Parent H2ClientProtocol instance + """ + self.stream_id: int = stream_id + self._request: Request = request + self._protocol: "H2ClientProtocol" = protocol + + self._download_maxsize = self._request.meta.get('download_maxsize', download_maxsize) + self._download_warnsize = self._request.meta.get('download_warnsize', download_warnsize) + + # Metadata of an HTTP/2 connection stream + # initialized when stream is instantiated + self.metadata: Dict = { + 'request_content_length': 0 if self._request.body is None else len(self._request.body), + + # Flag to keep track whether the stream has initiated the request + 'request_sent': False, + + # Flag to track whether we have logged about exceeding download warnsize + 'reached_warnsize': False, + + # Each time we send a data frame, we will decrease value by the amount send. + 'remaining_content_length': 0 if self._request.body is None else len(self._request.body), + + # Flag to keep track whether client (self) have closed this stream + 'stream_closed_local': False, + + # Flag to keep track whether the server has closed the stream + 'stream_closed_server': False, + } + + # Private variable used to build the response + # this response is then converted to appropriate Response class + # passed to the response deferred callback + self._response: Dict = { + # Data received frame by frame from the server is appended + # and passed to the response Deferred when completely received. + 'body': BytesIO(), + + # The amount of data received that counts against the + # flow control window + 'flow_controlled_size': 0, + + # Headers received after sending the request + 'headers': Headers({}), + } + + def _cancel(_) -> None: + # Close this stream as gracefully as possible + # If the associated request is initiated we reset this stream + # else we directly call close() method + if self.metadata['request_sent']: + self.reset_stream(StreamCloseReason.CANCELLED) + else: + self.close(StreamCloseReason.CANCELLED) + + self._deferred_response = Deferred(_cancel) + + def __str__(self) -> str: + return f'Stream(id={self.stream_id!r})' + + __repr__ = __str__ + + @property + def _log_warnsize(self) -> bool: + """Checks if we have received data which exceeds the download warnsize + and whether we have not already logged about it. + + Returns: + True if both the above conditions hold true + False if any of the conditions is false + """ + content_length_header = int(self._response['headers'].get(b'Content-Length', -1)) + return ( + self._download_warnsize + and ( + self._response['flow_controlled_size'] > self._download_warnsize + or content_length_header > self._download_warnsize + ) + and not self.metadata['reached_warnsize'] + ) + + def get_response(self) -> Deferred: + """Simply return a Deferred which fires when response + from the asynchronous request is available + """ + return self._deferred_response + + def check_request_url(self) -> bool: + # Make sure that we are sending the request to the correct URL + url = urlparse(self._request.url) + return ( + url.netloc == str(self._protocol.metadata['uri'].host, 'utf-8') + or url.netloc == str(self._protocol.metadata['uri'].netloc, 'utf-8') + or url.netloc == f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' + ) + + def _get_request_headers(self) -> List[Tuple[str, str]]: + url = urlparse(self._request.url) + + path = url.path + if url.query: + path += '?' + url.query + + # This pseudo-header field MUST NOT be empty for "http" or "https" + # URIs; "http" or "https" URIs that do not contain a path component + # MUST include a value of '/'. The exception to this rule is an + # OPTIONS request for an "http" or "https" URI that does not include + # a path component; these MUST include a ":path" pseudo-header field + # with a value of '*' (refer RFC 7540 - Section 8.1.2.3) + if not path: + path = '*' if self._request.method == 'OPTIONS' else '/' + + # Make sure pseudo-headers comes before all the other headers + headers = [ + (':method', self._request.method), + (':authority', url.netloc), + ] + + # The ":scheme" and ":path" pseudo-header fields MUST + # be omitted for CONNECT method (refer RFC 7540 - Section 8.3) + if self._request.method != 'CONNECT': + headers += [ + (':scheme', self._protocol.metadata['uri'].scheme), + (':path', path), + ] + + content_length = str(len(self._request.body)) + headers.append(('Content-Length', content_length)) + + content_length_name = self._request.headers.normkey(b'Content-Length') + for name, values in self._request.headers.items(): + for value in values: + value = str(value, 'utf-8') + if name == content_length_name: + if value != content_length: + logger.warning( + 'Ignoring bad Content-Length header %r of request %r, ' + 'sending %r instead', + value, + self._request, + content_length, + ) + continue + headers.append((str(name, 'utf-8'), value)) + + return headers + + def initiate_request(self) -> None: + if self.check_request_url(): + headers = self._get_request_headers() + self._protocol.conn.send_headers(self.stream_id, headers, end_stream=False) + self.metadata['request_sent'] = True + self.send_data() + else: + # Close this stream calling the response errback + # Note that we have not sent any headers + self.close(StreamCloseReason.INVALID_HOSTNAME) + + def send_data(self) -> None: + """Called immediately after the headers are sent. Here we send all the + data as part of the request. + + If the content length is 0 initially then we end the stream immediately and + wait for response data. + + Warning: Only call this method when stream not closed from client side + and has initiated request already by sending HEADER frame. If not then + stream will raise ProtocolError (raise by h2 state machine). + """ + if self.metadata['stream_closed_local']: + raise StreamClosedError(self.stream_id) + + # Firstly, check what the flow control window is for current stream. + window_size = self._protocol.conn.local_flow_control_window(stream_id=self.stream_id) + + # Next, check what the maximum frame size is. + max_frame_size = self._protocol.conn.max_outbound_frame_size + + # We will send no more than the window size or the remaining file size + # of data in this call, whichever is smaller. + bytes_to_send_size = min(window_size, self.metadata['remaining_content_length']) + + # We now need to send a number of data frames. + while bytes_to_send_size > 0: + chunk_size = min(bytes_to_send_size, max_frame_size) + + data_chunk_start_id = self.metadata['request_content_length'] - self.metadata['remaining_content_length'] + data_chunk = self._request.body[data_chunk_start_id:data_chunk_start_id + chunk_size] + + self._protocol.conn.send_data(self.stream_id, data_chunk, end_stream=False) + + bytes_to_send_size -= chunk_size + self.metadata['remaining_content_length'] -= chunk_size + + self.metadata['remaining_content_length'] = max(0, self.metadata['remaining_content_length']) + + # End the stream if no more data needs to be send + if self.metadata['remaining_content_length'] == 0: + self._protocol.conn.end_stream(self.stream_id) + + # Q. What about the rest of the data? + # Ans: Remaining Data frames will be sent when we get a WindowUpdate frame + + def receive_window_update(self) -> None: + """Flow control window size was changed. + Send data that earlier could not be sent as we were + blocked behind the flow control. + """ + if ( + self.metadata['remaining_content_length'] + and not self.metadata['stream_closed_server'] + and self.metadata['request_sent'] + ): + self.send_data() + + def receive_data(self, data: bytes, flow_controlled_length: int) -> None: + self._response['body'].write(data) + self._response['flow_controlled_size'] += flow_controlled_length + + # We check maxsize here in case the Content-Length header was not received + if self._download_maxsize and self._response['flow_controlled_size'] > self._download_maxsize: + self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) + return + + if self._log_warnsize: + self.metadata['reached_warnsize'] = True + warning_msg = ( + f'Received more ({self._response["flow_controlled_size"]}) bytes than download ' + f'warn size ({self._download_warnsize}) in request {self._request}' + ) + logger.warning(warning_msg) + + # Acknowledge the data received + self._protocol.conn.acknowledge_received_data( + self._response['flow_controlled_size'], + self.stream_id + ) + + def receive_headers(self, headers: List[HeaderTuple]) -> None: + for name, value in headers: + self._response['headers'][name] = value + + # Check if we exceed the allowed max data size which can be received + expected_size = int(self._response['headers'].get(b'Content-Length', -1)) + if self._download_maxsize and expected_size > self._download_maxsize: + self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED) + return + + if self._log_warnsize: + self.metadata['reached_warnsize'] = True + warning_msg = ( + f'Expected response size ({expected_size}) larger than ' + f'download warn size ({self._download_warnsize}) in request {self._request}' + ) + logger.warning(warning_msg) + + def reset_stream(self, reason: StreamCloseReason = StreamCloseReason.RESET) -> None: + """Close this stream by sending a RST_FRAME to the remote peer""" + if self.metadata['stream_closed_local']: + raise StreamClosedError(self.stream_id) + + # Clear buffer earlier to avoid keeping data in memory for a long time + self._response['body'].truncate(0) + + self.metadata['stream_closed_local'] = True + self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM) + self.close(reason) + + def close( + self, + reason: StreamCloseReason, + errors: Optional[List[BaseException]] = None, + from_protocol: bool = False, + ) -> None: + """Based on the reason sent we will handle each case. + """ + if self.metadata['stream_closed_server']: + raise StreamClosedError(self.stream_id) + + if not isinstance(reason, StreamCloseReason): + raise TypeError(f'Expected StreamCloseReason, received {reason.__class__.__qualname__}') + + # Have default value of errors as an empty list as + # some cases can add a list of exceptions + errors = errors or [] + + if not from_protocol: + self._protocol.pop_stream(self.stream_id) + + self.metadata['stream_closed_server'] = True + + # We do not check for Content-Length or Transfer-Encoding in response headers + # and add `partial` flag as in HTTP/1.1 as 'A request or response that includes + # a payload body can include a content-length header field' (RFC 7540 - Section 8.1.2.6) + + # NOTE: Order of handling the events is important here + # As we immediately cancel the request when maxsize is exceeded while + # receiving DATA_FRAME's when we have received the headers (not + # having Content-Length) + if reason is StreamCloseReason.MAXSIZE_EXCEEDED: + expected_size = int(self._response['headers'].get( + b'Content-Length', + self._response['flow_controlled_size']) + ) + error_msg = ( + f'Cancelling download of {self._request.url}: received response ' + f'size ({expected_size}) larger than download max size ({self._download_maxsize})' + ) + logger.error(error_msg) + self._deferred_response.errback(CancelledError(error_msg)) + + elif reason is StreamCloseReason.ENDED: + self._fire_response_deferred() + + # Stream was abruptly ended here + elif reason is StreamCloseReason.CANCELLED: + # Client has cancelled the request. Remove all the data + # received and fire the response deferred with no flags set + + # NOTE: The data is already flushed in Stream.reset_stream() called + # immediately when the stream needs to be cancelled + + # There maybe no :status in headers, we make + # HTTP Status Code: 499 - Client Closed Request + self._response['headers'][':status'] = '499' + self._fire_response_deferred() + + elif reason is StreamCloseReason.RESET: + self._deferred_response.errback(ResponseFailed([ + Failure( + f'Remote peer {self._protocol.metadata["ip_address"]} sent RST_STREAM', + ProtocolError + ) + ])) + + elif reason is StreamCloseReason.CONNECTION_LOST: + self._deferred_response.errback(ResponseFailed(errors)) + + elif reason is StreamCloseReason.INACTIVE: + errors.insert(0, InactiveStreamClosed(self._request)) + self._deferred_response.errback(ResponseFailed(errors)) + + else: + assert reason is StreamCloseReason.INVALID_HOSTNAME + self._deferred_response.errback(InvalidHostname( + self._request, + str(self._protocol.metadata['uri'].host, 'utf-8'), + f'{self._protocol.metadata["ip_address"]}:{self._protocol.metadata["uri"].port}' + )) + + def _fire_response_deferred(self) -> None: + """Builds response from the self._response dict + and fires the response deferred callback with the + generated response instance""" + + body = self._response['body'].getvalue() + response_cls = responsetypes.from_args( + headers=self._response['headers'], + url=self._request.url, + body=body, + ) + + response = response_cls( + url=self._request.url, + status=int(self._response['headers'][':status']), + headers=self._response['headers'], + body=body, + request=self._request, + certificate=self._protocol.metadata['certificate'], + ip_address=self._protocol.metadata['ip_address'], + protocol='h2', + ) + + self._deferred_response.callback(response) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a18c26b17..5ba0fb63b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,46 +1,179 @@ -import os import json import logging -import warnings -from os.path import join, exists +import os +from abc import abstractmethod +from os.path import exists, join +from typing import Optional, Type, TypeVar -from queuelib import PriorityQueue +from twisted.internet.defer import Deferred -from scrapy.utils.misc import load_object, create_instance +from scrapy.crawler import Crawler +from scrapy.http.request import Request +from scrapy.spiders import Spider from scrapy.utils.job import job_dir -from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.misc import create_instance, load_object logger = logging.getLogger(__name__) -class Scheduler: +class BaseSchedulerMeta(type): """ - Scrapy Scheduler. It allows to enqueue requests and then get - a next request to download. Scheduler is also handling duplication - filtering, via dupefilter. - - Prioritization and queueing is not performed by the Scheduler. - User sets ``priority`` field for each Request, and a PriorityQueue - (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities - to dequeue requests in a desired order. - - Scheduler uses two PriorityQueue instances, configured to work in-memory - and on-disk (optional). When on-disk queue is present, it is used by - default, and an in-memory queue is used as a fallback for cases where - a disk queue can't handle a request (can't serialize it). - - :setting:`SCHEDULER_MEMORY_QUEUE` and - :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes - which PriorityQueue instances would be instantiated with, to keep requests - on disk and in memory respectively. - - Overall, Scheduler is an object which holds several PriorityQueue instances - (in-memory and on-disk) and implements fallback logic for them. - Also, it handles dupefilters. + Metaclass to check scheduler classes against the necessary interface """ - def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, - logunser=False, stats=None, pqclass=None, crawler=None): + def __instancecheck__(cls, instance): + return cls.__subclasscheck__(type(instance)) + + def __subclasscheck__(cls, subclass): + return ( + hasattr(subclass, "has_pending_requests") and callable(subclass.has_pending_requests) + and hasattr(subclass, "enqueue_request") and callable(subclass.enqueue_request) + and hasattr(subclass, "next_request") and callable(subclass.next_request) + ) + + +class BaseScheduler(metaclass=BaseSchedulerMeta): + """ + The scheduler component is responsible for storing requests received from + the engine, and feeding them back upon request (also to the engine). + + The original sources of said requests are: + + * Spider: ``start_requests`` method, requests created for URLs in the ``start_urls`` attribute, request callbacks + * Spider middleware: ``process_spider_output`` and ``process_spider_exception`` methods + * Downloader middleware: ``process_request``, ``process_response`` and ``process_exception`` methods + + The order in which the scheduler returns its stored requests (via the ``next_request`` method) + plays a great part in determining the order in which those requests are downloaded. + + The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. + """ + + @classmethod + def from_crawler(cls, crawler: Crawler): + """ + Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument. + """ + return cls() + + def open(self, spider: Spider) -> Optional[Deferred]: + """ + Called when the spider is opened by the engine. It receives the spider + instance as argument and it's useful to execute initialization code. + + :param spider: the spider object for the current crawl + :type spider: :class:`~scrapy.spiders.Spider` + """ + pass + + def close(self, reason: str) -> Optional[Deferred]: + """ + Called when the spider is closed by the engine. It receives the reason why the crawl + finished as argument and it's useful to execute cleaning code. + + :param reason: a string which describes the reason why the spider was closed + :type reason: :class:`str` + """ + pass + + @abstractmethod + def has_pending_requests(self) -> bool: + """ + ``True`` if the scheduler has enqueued requests, ``False`` otherwise + """ + raise NotImplementedError() + + @abstractmethod + def enqueue_request(self, request: Request) -> bool: + """ + Process a request received by the engine. + + Return ``True`` if the request is stored correctly, ``False`` otherwise. + + If ``False``, the engine will fire a ``request_dropped`` signal, and + will not make further attempts to schedule the request at a later time. + For reference, the default Scrapy scheduler returns ``False`` when the + request is rejected by the dupefilter. + """ + raise NotImplementedError() + + @abstractmethod + def next_request(self) -> Optional[Request]: + """ + Return the next :class:`~scrapy.http.Request` to be processed, or ``None`` + to indicate that there are no requests to be considered ready at the moment. + + Returning ``None`` implies that no request from the scheduler will be sent + to the downloader in the current reactor cycle. The engine will continue + calling ``next_request`` until ``has_pending_requests`` is ``False``. + """ + raise NotImplementedError() + + +SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler") + + +class Scheduler(BaseScheduler): + """ + Default Scrapy scheduler. This implementation also handles duplication + filtering via the :setting:`dupefilter `. + + This scheduler stores requests into several priority queues (defined by the + :setting:`SCHEDULER_PRIORITY_QUEUE` setting). In turn, said priority queues + are backed by either memory or disk based queues (respectively defined by the + :setting:`SCHEDULER_MEMORY_QUEUE` and :setting:`SCHEDULER_DISK_QUEUE` settings). + + Request prioritization is almost entirely delegated to the priority queue. The only + prioritization performed by this scheduler is using the disk-based queue if present + (i.e. if the :setting:`JOBDIR` setting is defined) and falling back to the memory-based + queue if a serialization error occurs. If the disk queue is not present, the memory one + is used directly. + + :param dupefilter: An object responsible for checking and filtering duplicate requests. + The value for the :setting:`DUPEFILTER_CLASS` setting is used by default. + :type dupefilter: :class:`scrapy.dupefilters.BaseDupeFilter` instance or similar: + any class that implements the `BaseDupeFilter` interface + + :param jobdir: The path of a directory to be used for persisting the crawl's state. + The value for the :setting:`JOBDIR` setting is used by default. + See :ref:`topics-jobs`. + :type jobdir: :class:`str` or ``None`` + + :param dqclass: A class to be used as persistent request queue. + The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default. + :type dqclass: class + + :param mqclass: A class to be used as non-persistent request queue. + The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default. + :type mqclass: class + + :param logunser: A boolean that indicates whether or not unserializable requests should be logged. + The value for the :setting:`SCHEDULER_DEBUG` setting is used by default. + :type logunser: bool + + :param stats: A stats collector object to record stats about the request scheduling process. + The value for the :setting:`STATS_CLASS` setting is used by default. + :type stats: :class:`scrapy.statscollectors.StatsCollector` instance or similar: + any class that implements the `StatsCollector` interface + + :param pqclass: A class to be used as priority queue for requests. + The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default. + :type pqclass: class + + :param crawler: The crawler object corresponding to the current crawl. + :type crawler: :class:`scrapy.crawler.Crawler` + """ + def __init__( + self, + dupefilter, + jobdir: Optional[str] = None, + dqclass=None, + mqclass=None, + logunser: bool = False, + stats=None, + pqclass=None, + crawler: Optional[Crawler] = None, + ): self.df = dupefilter self.dqdir = self._dqdir(jobdir) self.pqclass = pqclass @@ -51,42 +184,57 @@ class Scheduler: self.crawler = crawler @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings - dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) - dupefilter = create_instance(dupefilter_cls, settings, crawler) - pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) - if pqclass is PriorityQueue: - warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'" - " is no longer supported because of API changes; " - "please use 'scrapy.pqueues.ScrapyPriorityQueue'", - ScrapyDeprecationWarning) - from scrapy.pqueues import ScrapyPriorityQueue - pqclass = ScrapyPriorityQueue + def from_crawler(cls: Type[SchedulerTV], crawler) -> SchedulerTV: + """ + Factory method, initializes the scheduler with arguments taken from the crawl settings + """ + dupefilter_cls = load_object(crawler.settings['DUPEFILTER_CLASS']) + return cls( + dupefilter=create_instance(dupefilter_cls, crawler.settings, crawler), + jobdir=job_dir(crawler.settings), + dqclass=load_object(crawler.settings['SCHEDULER_DISK_QUEUE']), + mqclass=load_object(crawler.settings['SCHEDULER_MEMORY_QUEUE']), + logunser=crawler.settings.getbool('SCHEDULER_DEBUG'), + stats=crawler.stats, + pqclass=load_object(crawler.settings['SCHEDULER_PRIORITY_QUEUE']), + crawler=crawler, + ) - dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) - mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) - logunser = settings.getbool('SCHEDULER_DEBUG') - return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, - stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, - mqclass=mqclass, crawler=crawler) - - def has_pending_requests(self): + def has_pending_requests(self) -> bool: return len(self) > 0 - def open(self, spider): + def open(self, spider: Spider) -> Optional[Deferred]: + """ + (1) initialize the memory queue + (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory + (3) return the result of the dupefilter's ``open`` method + """ self.spider = spider self.mqs = self._mq() self.dqs = self._dq() if self.dqdir else None return self.df.open() - def close(self, reason): - if self.dqs: + def close(self, reason: str) -> Optional[Deferred]: + """ + (1) dump pending requests to disk if there is a disk queue + (2) return the result of the dupefilter's ``close`` method + """ + if self.dqs is not None: state = self.dqs.close() + assert isinstance(self.dqdir, str) self._write_dqs_state(self.dqdir, state) return self.df.close(reason) - def enqueue_request(self, request): + def enqueue_request(self, request: Request) -> bool: + """ + Unless the received request is filtered out by the Dupefilter, attempt to push + it into the disk queue, falling back to pushing it into the memory queue. + + Increment the appropriate stats, such as: ``scheduler/enqueued``, + ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``. + + Return ``True`` if the request was stored successfully, ``False`` otherwise. + """ if not request.dont_filter and self.df.request_seen(request): self.df.log(request, self.spider) return False @@ -99,24 +247,35 @@ class Scheduler: self.stats.inc_value('scheduler/enqueued', spider=self.spider) return True - def next_request(self): + def next_request(self) -> Optional[Request]: + """ + Return a :class:`~scrapy.http.Request` object from the memory queue, + falling back to the disk queue if the memory queue is empty. + Return ``None`` if there are no more enqueued requests. + + Increment the appropriate stats, such as: ``scheduler/dequeued``, + ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. + """ request = self.mqs.pop() - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider) else: request = self._dqpop() - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued/disk', spider=self.spider) - if request: + if request is not None: self.stats.inc_value('scheduler/dequeued', spider=self.spider) return request - def __len__(self): - return len(self.dqs) + len(self.mqs) if self.dqs else len(self.mqs) + def __len__(self) -> int: + """ + Return the total amount of enqueued requests + """ + return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs) - def _dqpush(self, request): + def _dqpush(self, request: Request) -> bool: if self.dqs is None: - return + return False try: self.dqs.push(request) except ValueError as e: # non serializable request @@ -127,18 +286,18 @@ class Scheduler: logger.warning(msg, {'request': request, 'reason': e}, exc_info=True, extra={'spider': self.spider}) self.logunser = False - self.stats.inc_value('scheduler/unserializable', - spider=self.spider) - return + self.stats.inc_value('scheduler/unserializable', spider=self.spider) + return False else: return True - def _mqpush(self, request): + def _mqpush(self, request: Request) -> None: self.mqs.push(request) - def _dqpop(self): - if self.dqs: + def _dqpop(self) -> Optional[Request]: + if self.dqs is not None: return self.dqs.pop() + return None def _mq(self): """ Create a new priority queue instance, with in-memory storage """ @@ -162,21 +321,22 @@ class Scheduler: {'queuesize': len(q)}, extra={'spider': self.spider}) return q - def _dqdir(self, jobdir): + def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: """ Return a folder name to keep disk queue state at """ - if jobdir: + if jobdir is not None: dqdir = join(jobdir, 'requests.queue') if not exists(dqdir): os.makedirs(dqdir) return dqdir + return None - def _read_dqs_state(self, dqdir): + def _read_dqs_state(self, dqdir: str) -> list: path = join(dqdir, 'active.json') if not exists(path): - return () + return [] with open(path) as f: return json.load(f) - def _write_dqs_state(self, dqdir, state): + def _write_dqs_state(self, dqdir: str, state: list) -> None: with open(join(dqdir, 'active.json'), 'w') as f: json.dump(state, f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 0d3e3450f..f40bccbb3 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -3,12 +3,13 @@ extracts information from them""" import logging from collections import deque +from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union from itemadapter import is_item -from twisted.internet import defer +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure -from scrapy import signals +from scrapy import signals, Spider from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response @@ -18,6 +19,9 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output +QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] + + logger = logging.getLogger(__name__) @@ -26,46 +30,46 @@ class Slot: MIN_RESPONSE_SIZE = 1024 - def __init__(self, max_active_size=5000000): + def __init__(self, max_active_size: int = 5000000): self.max_active_size = max_active_size - self.queue = deque() - self.active = set() - self.active_size = 0 - self.itemproc_size = 0 - self.closing = None + self.queue: Deque[QueueTuple] = deque() + self.active: Set[Request] = set() + self.active_size: int = 0 + self.itemproc_size: int = 0 + self.closing: Optional[Deferred] = None - def add_response_request(self, response, request): - deferred = defer.Deferred() - self.queue.append((response, request, deferred)) - if isinstance(response, Response): - self.active_size += max(len(response.body), self.MIN_RESPONSE_SIZE) + def add_response_request(self, result: Union[Response, Failure], request: Request) -> Deferred: + deferred = Deferred() + self.queue.append((result, request, deferred)) + if isinstance(result, Response): + self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE) else: self.active_size += self.MIN_RESPONSE_SIZE return deferred - def next_response_request_deferred(self): + def next_response_request_deferred(self) -> QueueTuple: response, request, deferred = self.queue.popleft() self.active.add(request) return response, request, deferred - def finish_response(self, response, request): + def finish_response(self, result: Union[Response, Failure], request: Request) -> None: self.active.remove(request) - if isinstance(response, Response): - self.active_size -= max(len(response.body), self.MIN_RESPONSE_SIZE) + if isinstance(result, Response): + self.active_size -= max(len(result.body), self.MIN_RESPONSE_SIZE) else: self.active_size -= self.MIN_RESPONSE_SIZE - def is_idle(self): + def is_idle(self) -> bool: return not (self.queue or self.active) - def needs_backout(self): + def needs_backout(self) -> bool: return self.active_size > self.max_active_size class Scraper: def __init__(self, crawler): - self.slot = None + self.slot: Optional[Slot] = None self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR']) self.itemproc = itemproc_cls.from_crawler(crawler) @@ -74,36 +78,39 @@ class Scraper: self.signals = crawler.signals self.logformatter = crawler.logformatter - @defer.inlineCallbacks - def open_spider(self, spider): + @inlineCallbacks + def open_spider(self, spider: Spider): """Open the given spider for scraping and allocate resources for it""" self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE')) yield self.itemproc.open_spider(spider) - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> Deferred: """Close a spider being scraped and release its resources""" - slot = self.slot - slot.closing = defer.Deferred() - slot.closing.addCallback(self.itemproc.close_spider) - self._check_if_closing(spider, slot) - return slot.closing + if self.slot is None: + raise RuntimeError("Scraper slot not assigned") + self.slot.closing = Deferred() + self.slot.closing.addCallback(self.itemproc.close_spider) + self._check_if_closing(spider) + return self.slot.closing - def is_idle(self): + def is_idle(self) -> bool: """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider, slot): - if slot.closing and slot.is_idle(): - slot.closing.callback(spider) + def _check_if_closing(self, spider: Spider) -> None: + assert self.slot is not None # typing + if self.slot.closing and self.slot.is_idle(): + self.slot.closing.callback(spider) - def enqueue_scrape(self, response, request, spider): - slot = self.slot - dfd = slot.add_response_request(response, request) + def enqueue_scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: + if self.slot is None: + raise RuntimeError("Scraper slot not assigned") + dfd = self.slot.add_response_request(result, request) def finish_scraping(_): - slot.finish_response(response, request) - self._check_if_closing(spider, slot) - self._scrape_next(spider, slot) + self.slot.finish_response(result, request) + self._check_if_closing(spider) + self._scrape_next(spider) return _ dfd.addBoth(finish_scraping) @@ -112,15 +119,16 @@ class Scraper: {'request': request}, exc_info=failure_to_exc_info(f), extra={'spider': spider})) - self._scrape_next(spider, slot) + self._scrape_next(spider) return dfd - def _scrape_next(self, spider, slot): - while slot.queue: - response, request, deferred = slot.next_response_request_deferred() + def _scrape_next(self, spider: Spider) -> None: + assert self.slot is not None # typing + while self.slot.queue: + response, request, deferred = self.slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, result, request, spider): + def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: """ Handle the downloaded response or failure through the spider callback/errback """ @@ -131,7 +139,7 @@ class Scraper: dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, result, request, spider): + def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: """ Handle the different cases of request's result been a Response or a Failure """ @@ -141,14 +149,14 @@ class Scraper: dfd = self.call_spider(result, request, spider) return dfd.addErrback(self._log_download_errors, result, request, spider) - def call_spider(self, result, request, spider): + def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request callback = result.request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) dfd = defer_succeed(result) - dfd.addCallback(callback, **result.request.cb_kwargs) + dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs) else: # result is a Failure result.request = request warn_on_generator_with_return_value(spider, request.errback) @@ -156,7 +164,7 @@ class Scraper: dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) - def handle_spider_error(self, _failure, request, response, spider): + def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None: exc = _failure.value if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') @@ -177,7 +185,7 @@ class Scraper: spider=spider ) - def handle_spider_output(self, result, request, response, spider): + def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred: if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) @@ -185,12 +193,14 @@ class Scraper: request, response, spider) return dfd - def _process_spidermw_output(self, output, request, response, spider): + def _process_spidermw_output(self, output: Any, request: Request, response: Response, + spider: Spider) -> Optional[Deferred]: """Process each Request/Item (given in the output parameter) returned from the given spider """ + assert self.slot is not None # typing if isinstance(output, Request): - self.crawler.engine.crawl(request=output, spider=spider) + self.crawler.engine.crawl(request=output) elif is_item(output): self.slot.itemproc_size += 1 dfd = self.itemproc.process_item(output, spider) @@ -205,12 +215,18 @@ class Scraper: {'request': request, 'typename': typename}, extra={'spider': spider}, ) + return None - def _log_download_errors(self, spider_failure, download_failure, request, spider): + def _log_download_errors(self, spider_failure: Failure, download_failure: Failure, request: Request, + spider: Spider) -> Union[Failure, None]: """Log and silence errors that come from the engine (typically download - errors that got propagated thru here) + errors that got propagated thru here). + + spider_failure: the value passed into the errback of self.call_spider() + download_failure: the value passed into _scrape2() from + ExecutionEngine._handle_downloader_output() as "result" """ - if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest): + if not download_failure.check(IgnoreRequest): if download_failure.frames: logkws = self.logformatter.download_error(download_failure, request, spider) logger.log( @@ -230,10 +246,12 @@ class Scraper: if spider_failure is not download_failure: return spider_failure + return None - def _itemproc_finished(self, output, item, response, spider): + def _itemproc_finished(self, output: Any, item: Any, response: Response, spider: Spider) -> None: """ItemProcessor finished for the given ``item`` and returned ``output`` """ + assert self.slot is not None # typing self.slot.itemproc_size -= 1 if isinstance(output, Failure): ex = output.value diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 763e0cdf6..7cdc28284 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,22 +4,25 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice +from typing import Any, Callable, Generator, Iterable, Union, cast +from twisted.internet.defer import Deferred from twisted.python.failure import Failure +from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput +from scrapy.http import Response from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableChain -def _isiterable(possible_iterator): - return hasattr(possible_iterator, '__iter__') +ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] -def _fname(f): - return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" +def _isiterable(o) -> bool: + return isinstance(o, Iterable) class SpiderMiddlewareManager(MiddlewareManager): @@ -41,88 +44,100 @@ class SpiderMiddlewareManager(MiddlewareManager): process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) - def scrape_response(self, scrape_func, response, request, spider): - - def process_spider_input(response): - for method in self.methods['process_spider_input']: - try: - result = method(response=response, spider=spider) - if result is not None: - msg = (f"Middleware {_fname(method)} must return None " - f"or raise an exception, got {type(result)}") - raise _InvalidOutput(msg) - except _InvalidOutput: - raise - except Exception: - return scrape_func(Failure(), request, spider) - return scrape_func(response, request, spider) - - def _evaluate_iterable(iterable, exception_processor_index, recover_to): + def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, + spider: Spider) -> Any: + for method in self.methods['process_spider_input']: + method = cast(Callable, method) try: - for r in iterable: - yield r + result = method(response=response, spider=spider) + if result is not None: + msg = (f"Middleware {method.__qualname__} must return None " + f"or raise an exception, got {type(result)}") + raise _InvalidOutput(msg) + except _InvalidOutput: + raise + except Exception: + return scrape_func(Failure(), request, spider) + return scrape_func(response, request, spider) + + def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable, + exception_processor_index: int, recover_to: MutableChain) -> Generator: + try: + for r in iterable: + yield r + except Exception as ex: + exception_result = self._process_spider_exception(response, spider, Failure(ex), + exception_processor_index) + if isinstance(exception_result, Failure): + raise + recover_to.extend(exception_result) + + def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, + start_index: int = 0) -> Union[Failure, MutableChain]: + exception = _failure.value + # don't handle _InvalidOutput exception + if isinstance(exception, _InvalidOutput): + return _failure + method_list = islice(self.methods['process_spider_exception'], start_index, None) + for method_index, method in enumerate(method_list, start=start_index): + if method is None: + continue + result = method(response=response, exception=exception, spider=spider) + if _isiterable(result): + # stop exception handling by handing control over to the + # process_spider_output chain if an iterable has been returned + return self._process_spider_output(response, spider, result, method_index + 1) + elif result is None: + continue + else: + msg = (f"Middleware {method.__qualname__} must return None " + f"or an iterable, got {type(result)}") + raise _InvalidOutput(msg) + return _failure + + def _process_spider_output(self, response: Response, spider: Spider, + result: Iterable, start_index: int = 0) -> MutableChain: + # items in this iterable do not need to go through the process_spider_output + # chain, they went through it already from the process_spider_exception method + recovered = MutableChain() + + method_list = islice(self.methods['process_spider_output'], start_index, None) + for method_index, method in enumerate(method_list, start=start_index): + if method is None: + continue + try: + # might fail directly if the output value is not a generator + result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = process_spider_exception(Failure(ex), exception_processor_index) + exception_result = self._process_spider_exception(response, spider, Failure(ex), method_index + 1) if isinstance(exception_result, Failure): raise - recover_to.extend(exception_result) + return exception_result + if _isiterable(result): + result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) + else: + msg = (f"Middleware {method.__qualname__} must return an " + f"iterable, got {type(result)}") + raise _InvalidOutput(msg) - def process_spider_exception(_failure, start_index=0): - exception = _failure.value - # don't handle _InvalidOutput exception - if isinstance(exception, _InvalidOutput): - return _failure - method_list = islice(self.methods['process_spider_exception'], start_index, None) - for method_index, method in enumerate(method_list, start=start_index): - if method is None: - continue - result = method(response=response, exception=exception, spider=spider) - if _isiterable(result): - # stop exception handling by handing control over to the - # process_spider_output chain if an iterable has been returned - return process_spider_output(result, method_index + 1) - elif result is None: - continue - else: - msg = (f"Middleware {_fname(method)} must return None " - f"or an iterable, got {type(result)}") - raise _InvalidOutput(msg) - return _failure + return MutableChain(result, recovered) - def process_spider_output(result, start_index=0): - # items in this iterable do not need to go through the process_spider_output - # chain, they went through it already from the process_spider_exception method - recovered = MutableChain() + def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain: + recovered = MutableChain() + result = self._evaluate_iterable(response, spider, result, 0, recovered) + return MutableChain(self._process_spider_output(response, spider, result), recovered) - method_list = islice(self.methods['process_spider_output'], start_index, None) - for method_index, method in enumerate(method_list, start=start_index): - if method is None: - continue - try: - # might fail directly if the output value is not a generator - result = method(response=response, result=result, spider=spider) - except Exception as ex: - exception_result = process_spider_exception(Failure(ex), method_index + 1) - if isinstance(exception_result, Failure): - raise - return exception_result - if _isiterable(result): - result = _evaluate_iterable(result, method_index + 1, recovered) - else: - msg = (f"Middleware {_fname(method)} must return an " - f"iterable, got {type(result)}") - raise _InvalidOutput(msg) + def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, + spider: Spider) -> Deferred: + def process_callback_output(result: Iterable) -> MutableChain: + return self._process_callback_output(response, spider, result) - return MutableChain(result, recovered) + def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: + return self._process_spider_exception(response, spider, _failure) - def process_callback_output(result): - recovered = MutableChain() - result = _evaluate_iterable(result, 0, recovered) - return MutableChain(process_spider_output(result), recovered) - - dfd = mustbe_deferred(process_spider_input, response) + dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd - def process_start_requests(self, start_requests, spider): + def process_start_requests(self, start_requests, spider: Spider) -> Deferred: return self._process_chain('process_start_requests', start_requests, spider) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 578016536..fdca7b335 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -25,6 +25,7 @@ from scrapy.utils.log import ( configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, + log_reactor_info, log_scrapy_info, LogCounterHandler, ) @@ -38,7 +39,7 @@ logger = logging.getLogger(__name__) class Crawler: - def __init__(self, spidercls, settings=None): + def __init__(self, spidercls, settings=None, init_reactor: bool = False): if isinstance(spidercls, Spider): raise ValueError('The spidercls argument must be a class, not an object') @@ -50,6 +51,7 @@ class Crawler: self.spidercls.update_settings(self.settings) self.signals = SignalManager(self) + self.stats = load_object(self.settings['STATS_CLASS'])(self) handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL')) @@ -69,6 +71,26 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + + self.request_fingerprinter = create_instance( + load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']), + settings=self.settings, + crawler=self, + ) + + reactor_class = self.settings.get("TWISTED_REACTOR") + if init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if reactor_class: + install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) + else: + from twisted.internet import default + default.install() + log_reactor_info() + if reactor_class: + verify_installed_reactor(reactor_class) + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() @@ -153,7 +175,6 @@ class CrawlerRunner: self._crawlers = set() self._active = set() self.bootstrap_failed = False - self._handle_twisted_reactor() @property def spiders(self): @@ -247,10 +268,6 @@ class CrawlerRunner: while self._active: yield defer.DeferredList(self._active) - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - verify_installed_reactor(self.settings["TWISTED_REACTOR"]) - class CrawlerProcess(CrawlerRunner): """ @@ -278,7 +295,6 @@ class CrawlerProcess(CrawlerRunner): def __init__(self, settings=None, install_root_handler=True): super().__init__(settings) - install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) @@ -298,7 +314,12 @@ class CrawlerProcess(CrawlerRunner): {'signame': signame}) reactor.callFromThread(self._stop_reactor) - def start(self, stop_after_crawl=True): + def _create_crawler(self, spidercls): + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings, init_reactor=True) + + def start(self, stop_after_crawl=True, install_signal_handlers=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -309,6 +330,9 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished + + :param bool install_signal_handlers: whether to install the shutdown + handlers (default: True) """ from twisted.internet import reactor if stop_after_crawl: @@ -318,6 +342,8 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) + if install_signal_handlers: + install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() @@ -337,8 +363,3 @@ class CrawlerProcess(CrawlerRunner): reactor.stop() except RuntimeError: # raised if already stopped or in shutdown stage pass - - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) - super()._handle_twisted_reactor() diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d95ed3d38..3afa06077 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,15 +1,26 @@ import logging from collections import defaultdict +from tldextract import TLDExtract + from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) +_split_domain = TLDExtract(include_psl_private_domains=True) + + +def _is_public_domain(domain): + parts = _split_domain(domain) + return not parts.domain + + class CookiesMiddleware: """This middleware enables working with sites that need cookies""" @@ -23,14 +34,29 @@ class CookiesMiddleware: raise NotConfigured return cls(crawler.settings.getbool('COOKIES_DEBUG')) + def _process_cookies(self, cookies, *, jar, request): + for cookie in cookies: + cookie_domain = cookie.domain + if cookie_domain.startswith('.'): + cookie_domain = cookie_domain[1:] + + request_domain = urlparse_cached(request).hostname.lower() + + if cookie_domain and _is_public_domain(cookie_domain): + if cookie_domain != request_domain: + continue + cookie.domain = request_domain + + jar.set_cookie_if_ok(cookie, request) + def process_request(self, request, spider): if request.meta.get('dont_merge_cookies', False): return cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] - for cookie in self._get_request_cookies(jar, request): - jar.set_cookie_if_ok(cookie, request) + cookies = self._get_request_cookies(jar, request) + self._process_cookies(cookies, jar=jar, request=request) # set Cookie header request.headers.pop('Cookie', None) @@ -44,7 +70,9 @@ class CookiesMiddleware: # extract cookies from Set-Cookie and drop invalid/expired cookies cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] - jar.extract_cookies(response, request) + cookies = jar.make_cookies(response, request) + self._process_cookies(cookies, jar=jar, request=request) + self._debug_set_cookie(response, spider) return response @@ -80,8 +108,8 @@ class CookiesMiddleware: logger.warning(msg.format(request, cookie, key)) return continue - if isinstance(cookie[key], str): - decoded[key] = cookie[key] + if isinstance(cookie[key], (bool, float, int, str)): + decoded[key] = str(cookie[key]) else: try: decoded[key] = cookie[key].decode("utf8") diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 089bf0d85..1bee3e279 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -3,10 +3,14 @@ HTTP basic auth downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +import warnings from w3lib.http import basic_auth_header from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.url import url_is_from_any_domain class HttpAuthMiddleware: @@ -24,8 +28,23 @@ class HttpAuthMiddleware: pwd = getattr(spider, 'http_pass', '') if usr or pwd: self.auth = basic_auth_header(usr, pwd) + if not hasattr(spider, 'http_auth_domain'): + warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security ' + 'problems if the spider makes requests to several different domains. http_auth_domain ' + 'will be set to the domain of the first request, please set it to the correct value ' + 'explicitly.', + category=ScrapyDeprecationWarning) + self.domain_unset = True + else: + self.domain = spider.http_auth_domain + self.domain_unset = False def process_request(self, request, spider): auth = getattr(self, 'auth', None) if auth and b'Authorization' not in request.headers: - request.headers[b'Authorization'] = auth + domain = urlparse_cached(request).hostname + if self.domain_unset: + self.domain = domain + self.domain_unset = False + if not self.domain or url_is_from_any_domain(request.url, [self.domain]): + request.headers[b'Authorization'] = auth diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 62f1c3a29..80ed7ac75 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -70,7 +70,7 @@ class HttpCacheMiddleware: self.stats.inc_value('httpcache/miss', spider=spider) if self.ignore_missing: self.stats.inc_value('httpcache/ignore', spider=spider) - raise IgnoreRequest("Ignored request not in cache: %s" % request) + raise IgnoreRequest(f"Ignored request not in cache: {request}") return None # first time request # Return cached response only if not expired diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f504302e2..4e7feeeaf 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,10 +1,12 @@ import io +import warnings import zlib -from scrapy.utils.gz import gunzip +from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.exceptions import NotConfigured +from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.gz import gunzip ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] @@ -25,11 +27,25 @@ except ImportError: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" + def __init__(self, stats=None): + self.stats = stats + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured - return cls() + try: + return cls(stats=crawler.stats) + except TypeError: + warnings.warn( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'stats' parameter or " + "reimplement the 'from_crawler' method.", + ScrapyDeprecationWarning, + ) + result = cls() + result.stats = crawler.stats + return result def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', @@ -44,6 +60,9 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) + if self.stats: + self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + self.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 4053fecc5..c8c84ffb2 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -4,6 +4,7 @@ from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string from scrapy.http import HtmlResponse +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured @@ -11,6 +12,20 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) +def _build_redirect_request(source_request, *, url, **kwargs): + redirect_request = source_request.replace( + url=url, + **kwargs, + cookies=None, + ) + if 'Cookie' in redirect_request.headers: + source_request_netloc = urlparse_cached(source_request).netloc + redirect_request_netloc = urlparse_cached(redirect_request).netloc + if source_request_netloc != redirect_request_netloc: + del redirect_request.headers['Cookie'] + return redirect_request + + class BaseRedirectMiddleware: enabled_setting = 'REDIRECT_ENABLED' @@ -47,10 +62,15 @@ class BaseRedirectMiddleware: raise IgnoreRequest("max redirections reached") def _redirect_request_using_get(self, request, redirect_url): - redirected = request.replace(url=redirect_url, method='GET', body='') - redirected.headers.pop('Content-Type', None) - redirected.headers.pop('Content-Length', None) - return redirected + redirect_request = _build_redirect_request( + request, + url=redirect_url, + method='GET', + body='', + ) + redirect_request.headers.pop('Content-Type', None) + redirect_request.headers.pop('Content-Length', None) + return redirect_request class RedirectMiddleware(BaseRedirectMiddleware): @@ -80,7 +100,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): redirected_url = urljoin(request.url, location) if response.status in (301, 307, 308) or request.method == 'HEAD': - redirected = request.replace(url=redirected_url) + redirected = _build_redirect_request(request, url=redirected_url) return self._redirect(redirected, request, spider, response.status) redirected = self._redirect_request_using_get(request, redirected_url) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 51fe59254..c6cc7c56d 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -2,14 +2,15 @@ An extension to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. -You can change the behaviour of this middleware by modifing the scraping settings: +You can change the behaviour of this middleware by modifying the scraping settings: RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ -import logging +from logging import getLogger, Logger +from typing import Optional, Union from twisted.internet import defer from twisted.internet.error import ( @@ -23,12 +24,104 @@ from twisted.internet.error import ( ) from twisted.web.client import ResponseFailed -from scrapy.exceptions import NotConfigured -from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError +from scrapy.exceptions import NotConfigured +from scrapy.http.request import Request +from scrapy.spiders import Spider from scrapy.utils.python import global_object_name +from scrapy.utils.response import response_status_message -logger = logging.getLogger(__name__) + +retry_logger = getLogger(__name__) + + +def get_retry_request( + request: Request, + *, + spider: Spider, + reason: Union[str, Exception] = 'unspecified', + max_retry_times: Optional[int] = None, + priority_adjust: Optional[int] = None, + logger: Logger = retry_logger, + stats_base_key: str = 'retry', +): + """ + Returns a new :class:`~scrapy.Request` object to retry the specified + request, or ``None`` if retries of the specified request have been + exhausted. + + For example, in a :class:`~scrapy.Spider` callback, you could use it as + follows:: + + def parse(self, response): + if not response.text: + new_request_or_none = get_retry_request( + response.request, + spider=self, + reason='empty', + ) + return new_request_or_none + + *spider* is the :class:`~scrapy.Spider` instance which is asking for the + retry request. It is used to access the :ref:`settings ` + and :ref:`stats `, and to provide extra logging context (see + :func:`logging.debug`). + + *reason* is a string or an :class:`Exception` object that indicates the + reason why the request needs to be retried. It is used to name retry stats. + + *max_retry_times* is a number that determines the maximum number of times + that *request* can be retried. If not specified or ``None``, the number is + read from the :reqmeta:`max_retry_times` meta key of the request. If the + :reqmeta:`max_retry_times` meta key is not defined or ``None``, the number + is read from the :setting:`RETRY_TIMES` setting. + + *priority_adjust* is a number that determines how the priority of the new + request changes in relation to *request*. If not specified, the number is + read from the :setting:`RETRY_PRIORITY_ADJUST` setting. + + *logger* is the logging.Logger object to be used when logging messages + + *stats_base_key* is a string to be used as the base key for the + retry-related job stats + """ + settings = spider.crawler.settings + stats = spider.crawler.stats + retry_times = request.meta.get('retry_times', 0) + 1 + if max_retry_times is None: + max_retry_times = request.meta.get('max_retry_times') + if max_retry_times is None: + max_retry_times = settings.getint('RETRY_TIMES') + if retry_times <= max_retry_times: + logger.debug( + "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider} + ) + new_request: Request = request.copy() + new_request.meta['retry_times'] = retry_times + new_request.dont_filter = True + if priority_adjust is None: + priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') + new_request.priority = request.priority + priority_adjust + + if callable(reason): + reason = reason() + if isinstance(reason, Exception): + reason = global_object_name(reason.__class__) + + stats.inc_value(f'{stats_base_key}/count') + stats.inc_value(f'{stats_base_key}/reason_count/{reason}') + return new_request + else: + stats.inc_value(f'{stats_base_key}/max_reached') + logger.error( + "Gave up retrying %(request)s (failed %(retry_times)d times): " + "%(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}, + ) + return None class RetryMiddleware: @@ -67,31 +160,12 @@ class RetryMiddleware: return self._retry(request, exception, spider) def _retry(self, request, reason, spider): - retries = request.meta.get('retry_times', 0) + 1 - - retry_times = self.max_retry_times - - if 'max_retry_times' in request.meta: - retry_times = request.meta['max_retry_times'] - - stats = spider.crawler.stats - if retries <= retry_times: - logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", - {'request': request, 'retries': retries, 'reason': reason}, - extra={'spider': spider}) - retryreq = request.copy() - retryreq.meta['retry_times'] = retries - retryreq.dont_filter = True - retryreq.priority = request.priority + self.priority_adjust - - if isinstance(reason, Exception): - reason = global_object_name(reason.__class__) - - stats.inc_value('retry/count') - stats.inc_value(f'retry/reason_count/{reason}') - return retryreq - else: - stats.inc_value('retry/max_reached') - logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", - {'request': request, 'retries': retries, 'reason': reason}, - extra={'spider': spider}) + max_retry_times = request.meta.get('max_retry_times', self.max_retry_times) + priority_adjust = request.meta.get('priority_adjust', self.priority_adjust) + return get_retry_request( + request, + reason=reason, + spider=spider, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index d6da55535..e66bf177e 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -67,7 +67,7 @@ class RobotsTxtMiddleware: priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - dfd = self.crawler.engine.download(robotsreq, spider) + dfd = self.crawler.engine.download(robotsreq) dfd.addCallback(self._parse_robots, netloc, spider) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 5479cd0e2..25fb1ed9d 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,7 +1,22 @@ from scrapy.exceptions import NotConfigured +from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr -from scrapy.utils.response import response_httprepr -from scrapy.utils.python import global_object_name + +from twisted.web import http + + +def get_header_size(headers): + size = 0 + for key, value in headers.items(): + if isinstance(value, (list, tuple)): + for v in value: + size += len(b": ") + len(key) + len(v) + return size + len(b'\r\n') * (len(headers.keys()) - 1) + + +def get_status_size(response_status): + return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15 + # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> " class DownloaderStats: @@ -24,7 +39,8 @@ class DownloaderStats: def process_response(self, request, response, spider): self.stats.inc_value('downloader/response_count', spider=spider) self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider) - reslen = len(response_httprepr(response)) + reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4 + # response.body + b"\r\n"+ response.header + b"\r\n" + response.status self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index ac5478e7c..d1b0559ef 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,35 +1,56 @@ -import os import logging +import os +from typing import Optional, Set, Type, TypeVar +from warnings import warn +from twisted.internet.defer import Deferred + +from scrapy.http.request import Request +from scrapy.settings import BaseSettings +from scrapy.spiders import Spider +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir -from scrapy.utils.request import referer_str, request_fingerprint +from scrapy.utils.request import referer_str, RequestFingerprinter + + +BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") class BaseDupeFilter: - @classmethod - def from_settings(cls, settings): + def from_settings(cls: Type[BaseDupeFilterTV], settings: BaseSettings) -> BaseDupeFilterTV: return cls() - def request_seen(self, request): + def request_seen(self, request: Request) -> bool: return False - def open(self): # can return deferred + def open(self) -> Optional[Deferred]: pass - def close(self, reason): # can return a deferred + def close(self, reason: str) -> Optional[Deferred]: pass - def log(self, request, spider): # log that a request has been filtered + def log(self, request: Request, spider: Spider) -> None: + """Log that a request has been filtered""" pass +RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter") + + class RFPDupeFilter(BaseDupeFilter): """Request Fingerprint duplicates filter""" - def __init__(self, path=None, debug=False): + def __init__( + self, + path: Optional[str] = None, + debug: bool = False, + *, + fingerprinter=None, + ) -> None: self.file = None - self.fingerprints = set() + self.fingerprinter = fingerprinter or RequestFingerprinter() + self.fingerprints: Set[str] = set() self.logdupes = True self.debug = debug self.logger = logging.getLogger(__name__) @@ -39,26 +60,57 @@ class RFPDupeFilter(BaseDupeFilter): self.fingerprints.update(x.rstrip() for x in self.file) @classmethod - def from_settings(cls, settings): + def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV: debug = settings.getbool('DUPEFILTER_DEBUG') - return cls(job_dir(settings), debug) + try: + return cls(job_dir(settings), debug, fingerprinter=fingerprinter) + except TypeError: + warn( + "RFPDupeFilter subclasses must either modify their '__init__' " + "method to support a 'fingerprinter' parameter or reimplement " + "the 'from_settings' class method.", + ScrapyDeprecationWarning, + ) + result = cls(job_dir(settings), debug) + result.fingerprinter = fingerprinter + return result - def request_seen(self, request): + @classmethod + def from_crawler(cls, crawler): + try: + return cls.from_settings( + crawler.settings, + fingerprinter=crawler.request_fingerprinter, + ) + except TypeError: + warn( + "RFPDupeFilter subclasses must either modify their overridden " + "'__init__' method and 'from_settings' class method to " + "support a 'fingerprinter' parameter, or reimplement the " + "'from_crawler' class method.", + ScrapyDeprecationWarning, + ) + result = cls.from_settings(crawler.settings) + result.fingerprinter = crawler.request_fingerprinter + return result + + def request_seen(self, request: Request) -> bool: fp = self.request_fingerprint(request) if fp in self.fingerprints: return True self.fingerprints.add(fp) if self.file: self.file.write(fp + '\n') + return False - def request_fingerprint(self, request): - return request_fingerprint(request) + def request_fingerprint(self, request: Request) -> str: + return self.fingerprinter.fingerprint(request).hex() - def close(self, reason): + def close(self, reason: str) -> None: if self.file: self.file.close() - def log(self, request, spider): + def log(self, request: Request, spider: Spider) -> None: if self.debug: msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)" args = {'request': request, 'referer': referer_str(request)} diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 691491fbd..ad12f26d6 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -14,7 +14,7 @@ from xml.sax.saxutils import XMLGenerator from itemadapter import is_item, ItemAdapter from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import _BaseItem +from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -31,7 +31,7 @@ class BaseItemExporter: self._configure(kwargs, dont_fail=dont_fail) def _configure(self, options, dont_fail=False): - """Configure the exporter by poping options from the ``options`` dict. + """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ @@ -332,7 +332,7 @@ class PythonItemExporter(BaseItemExporter): return serializer(value) def _serialize_value(self, value): - if isinstance(value, _BaseItem): + if isinstance(value, Item): return self.export_item(value) elif is_item(value): return dict(self._serialize_item(value)) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bec114707..e7097b7a1 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,14 +11,16 @@ import sys import warnings from datetime import datetime from tempfile import NamedTemporaryFile +from typing import Any, Callable, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads from w3lib.url import file_uri_to_path from zope.interface import implementer, Interface -from scrapy import signals +from scrapy import signals, Spider from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file @@ -36,16 +38,49 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): kwargs['feed_options'] = feed_options else: warnings.warn( - "{} does not support the 'feed_options' keyword argument. Add a " + f"{builder.__qualname__} does not support the 'feed_options' keyword argument. Add a " "'feed_options' parameter to its signature to remove this " "warning. This parameter will become mandatory in a future " - "version of Scrapy." - .format(builder.__qualname__), + "version of Scrapy.", category=ScrapyDeprecationWarning ) return builder(*preargs, uri, *args, **kwargs) +class ItemFilter: + """ + This will be used by FeedExporter to decide if an item should be allowed + to be exported to a particular feed. + + :param feed_options: feed specific options passed from FeedExporter + :type feed_options: dict + """ + feed_options: Optional[dict] + item_classes: Tuple + + def __init__(self, feed_options: Optional[dict]) -> None: + self.feed_options = feed_options + if feed_options is not None: + self.item_classes = tuple( + load_object(item_class) for item_class in feed_options.get("item_classes") or () + ) + else: + self.item_classes = tuple() + + def accepts(self, item: Any) -> bool: + """ + Return ``True`` if `item` should be exported or ``False`` otherwise. + + :param item: scraped item which user wants to check if is acceptable + :type item: :ref:`Scrapy items ` + :return: `True` if accepted, `False` otherwise + :rtype: bool + """ + if self.item_classes: + return isinstance(item, self.item_classes) + return True # accept all items by default + + class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" @@ -118,21 +153,25 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, - feed_options=None): + def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *, + feed_options=None, session_token=None): if not is_botocore_available(): raise NotConfigured('missing botocore library') u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key self.secret_key = u.password or secret_key + self.session_token = session_token self.keyname = u.path[1:] # remove first "/" self.acl = acl + self.endpoint_url = endpoint_url import botocore.session session = botocore.session.get_session() self.s3_client = session.create_client( 's3', aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key) + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url) if feed_options and feed_options.get('overwrite', True) is False: logger.warning('S3 does not support appending to files. To ' 'suppress this warning, remove the overwrite ' @@ -145,7 +184,9 @@ class S3FeedStorage(BlockingFeedStorage): uri, access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], + session_token=crawler.settings['AWS_SESSION_TOKEN'], acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, + endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None, feed_options=feed_options, ) @@ -215,7 +256,7 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter): self.file = file self.exporter = exporter self.storage = storage @@ -225,6 +266,7 @@ class _FeedSlot: self.store_empty = store_empty self.uri_template = uri_template self.uri = uri + self.filter = filter # flags self.itemcount = 0 self._exporting = False @@ -255,6 +297,7 @@ class FeedExporter: self.settings = crawler.settings self.feeds = {} self.slots = [] + self.filters = {} if not self.settings['FEEDS'] and not self.settings['FEED_URI']: raise NotConfigured @@ -269,12 +312,14 @@ class FeedExporter: uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.filters[uri] = self._load_filter(feed_options) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' for uri, feed_options in self.settings.getdict('FEEDS').items(): uri = str(uri) # handle pathlib.Path objects self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.filters[uri] = self._load_filter(feed_options) self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') @@ -310,32 +355,28 @@ class FeedExporter: # properly closed. return defer.maybeDeferred(slot.storage.store, slot.file) slot.finish_exporting() - logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" - log_args = {'format': slot.format, - 'itemcount': slot.itemcount, - 'uri': slot.uri} + logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" d = defer.maybeDeferred(slot.storage.store, slot.file) - # Use `largs=log_args` to copy log_args into function's scope - # instead of using `log_args` from the outer scope d.addCallback( - self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__ + self._handle_store_success, logmsg, spider, type(slot.storage).__name__ ) d.addErrback( - self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__ + self._handle_store_error, logmsg, spider, type(slot.storage).__name__ ) return d - def _handle_store_error(self, f, largs, logfmt, spider, slot_type): + def _handle_store_error(self, f, logmsg, spider, slot_type): logger.error( - logfmt % "Error storing", largs, + "Error storing %s", logmsg, exc_info=failure_to_exc_info(f), extra={'spider': spider} ) self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") - def _handle_store_success(self, f, largs, logfmt, spider, slot_type): + def _handle_store_success(self, f, logmsg, spider, slot_type): logger.info( - logfmt % "Stored", largs, extra={'spider': spider} + "Stored %s", logmsg, + extra={'spider': spider} ) self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") @@ -351,6 +392,9 @@ class FeedExporter: """ storage = self._get_storage(uri, feed_options) file = storage.open(spider) + if "postprocessing" in feed_options: + file = PostProcessingManager(feed_options["postprocessing"], file, feed_options) + exporter = self._get_exporter( file=file, format=feed_options['format'], @@ -368,6 +412,7 @@ class FeedExporter: store_empty=feed_options['store_empty'], batch_id=batch_id, uri_template=uri_template, + filter=self.filters[uri_template] ) if slot.store_empty: slot.start_exporting() @@ -376,6 +421,10 @@ class FeedExporter: def item_scraped(self, item, spider): slots = [] for slot in self.slots: + if not slot.filter.accepts(item): + slots.append(slot) # if slot doesn't accept item, continue with next slot + continue + slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 @@ -420,10 +469,10 @@ class FeedExporter: for uri_template, values in self.feeds.items(): if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT ' + '%%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT ' 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: ' - 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' - ''.format(uri_template) + 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count', + uri_template ) return False return True @@ -472,10 +521,15 @@ class FeedExporter: instance = build_instance(feedcls) method_name = '__new__' if instance is None: - raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name)) + raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") return instance - def _get_uri_params(self, spider, uri_params, slot=None): + def _get_uri_params( + self, + spider: Spider, + uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], + slot: Optional[_FeedSlot] = None, + ) -> dict: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -483,6 +537,20 @@ class FeedExporter: params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-') params['batch_time'] = utc_now.isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 - uripar_function = load_object(uri_params) if uri_params else lambda x, y: None - uripar_function(params, spider) - return params + original_params = params.copy() + uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params + new_params = uripar_function(params, spider) + if new_params is None or original_params != params: + warnings.warn( + 'Modifying the params dictionary in-place in the function defined in ' + 'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS ' + 'setting is deprecated. The function must return a new dictionary ' + 'instead.', + category=ScrapyDeprecationWarning + ) + return new_params if new_params is not None else params + + def _load_filter(self, feed_options): + # load the item filter if declared else load the default filter class + item_filter_class = load_object(feed_options.get("item_filter", ItemFilter)) + return item_filter_class(feed_options) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index e0c04b2de..c71484cfa 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.request import request_fingerprint logger = logging.getLogger(__name__) @@ -226,7 +225,9 @@ class DbmCacheStorage: dbpath = os.path.join(self.cachedir, f'{spider.name}.db') self.db = self.dbmodule.open(dbpath, 'c') - logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider}) + + self._fingerprinter = spider.crawler.request_fingerprinter def close_spider(self, spider): self.db.close() @@ -244,7 +245,7 @@ class DbmCacheStorage: return response def store_response(self, spider, request, response): - key = self._request_key(request) + key = self._fingerprinter.fingerprint(request).hex() data = { 'status': response.status, 'url': response.url, @@ -255,7 +256,7 @@ class DbmCacheStorage: self.db[f'{key}_time'] = str(time()) def _read_data(self, spider, request): - key = self._request_key(request) + key = self._fingerprinter.fingerprint(request).hex() db = self.db tkey = f'{key}_time' if tkey not in db: @@ -267,9 +268,6 @@ class DbmCacheStorage: return pickle.loads(db[f'{key}_data']) - def _request_key(self, request): - return request_fingerprint(request) - class FilesystemCacheStorage: @@ -280,9 +278,11 @@ class FilesystemCacheStorage: self._open = gzip.open if self.use_gzip else open def open_spider(self, spider): - logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}, + logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir}, extra={'spider': spider}) + self._fingerprinter = spider.crawler.request_fingerprinter + def close_spider(self, spider): pass @@ -329,7 +329,7 @@ class FilesystemCacheStorage: f.write(request.body) def _get_request_path(self, spider, request): - key = request_fingerprint(request) + key = self._fingerprinter.fingerprint(request).hex() return os.path.join(self.cachedir, spider.name, key[0:2], key) def _read_meta(self, spider, request): diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 274cbdbfe..f5081a7d7 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -33,8 +33,8 @@ class MemoryUsage: self.crawler = crawler self.warned = False self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL') - self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 - self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 + self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024 + self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024 self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS') self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) @@ -77,7 +77,7 @@ class MemoryUsage: def _check_limit(self): if self.get_virtual_size() > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) - mem = self.limit/1024/1024 + mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: @@ -88,19 +88,17 @@ class MemoryUsage: self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) - open_spiders = self.crawler.engine.open_spiders - if open_spiders: - for spider in open_spiders: - self.crawler.engine.close_spider(spider, 'memusage_exceeded') + if self.crawler.engine.spider is not None: + self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded') else: self.crawler.stop() def _check_warning(self): - if self.warned: # warn only once + if self.warned: # warn only once return if self.get_virtual_size() > self.warning: self.crawler.stats.set_value('memusage/warning_reached', 1) - mem = self.warning/1024/1024 + mem = self.warning / 1024 / 1024 logger.warning("Memory usage reached %(memusage)dM", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py new file mode 100644 index 000000000..413c2e55e --- /dev/null +++ b/scrapy/extensions/postprocessing.py @@ -0,0 +1,154 @@ +""" +Extension for processing data before they are exported to feeds. +""" +from bz2 import BZ2File +from gzip import GzipFile +from io import IOBase +from lzma import LZMAFile +from typing import Any, BinaryIO, Dict, List + +from scrapy.utils.misc import load_object + + +class GzipPlugin: + """ + Compresses received data using `gzip `_. + + Accepted ``feed_options`` parameters: + + - `gzip_compresslevel` + - `gzip_mtime` + - `gzip_filename` + + See :py:class:`gzip.GzipFile` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + compress_level = self.feed_options.get("gzip_compresslevel", 9) + mtime = self.feed_options.get("gzip_mtime") + filename = self.feed_options.get("gzip_filename") + self.gzipfile = GzipFile(fileobj=self.file, mode="wb", compresslevel=compress_level, + mtime=mtime, filename=filename) + + def write(self, data: bytes) -> int: + return self.gzipfile.write(data) + + def close(self) -> None: + self.gzipfile.close() + self.file.close() + + +class Bz2Plugin: + """ + Compresses received data using `bz2 `_. + + Accepted ``feed_options`` parameters: + + - `bz2_compresslevel` + + See :py:class:`bz2.BZ2File` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + compress_level = self.feed_options.get("bz2_compresslevel", 9) + self.bz2file = BZ2File(filename=self.file, mode="wb", compresslevel=compress_level) + + def write(self, data: bytes) -> int: + return self.bz2file.write(data) + + def close(self) -> None: + self.bz2file.close() + self.file.close() + + +class LZMAPlugin: + """ + Compresses received data using `lzma `_. + + Accepted ``feed_options`` parameters: + + - `lzma_format` + - `lzma_check` + - `lzma_preset` + - `lzma_filters` + + .. note:: + ``lzma_filters`` cannot be used in pypy version 7.3.1 and older. + + See :py:class:`lzma.LZMAFile` for more info about parameters. + """ + + def __init__(self, file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.file = file + self.feed_options = feed_options + + format = self.feed_options.get("lzma_format") + check = self.feed_options.get("lzma_check", -1) + preset = self.feed_options.get("lzma_preset") + filters = self.feed_options.get("lzma_filters") + self.lzmafile = LZMAFile(filename=self.file, mode="wb", format=format, + check=check, preset=preset, filters=filters) + + def write(self, data: bytes) -> int: + return self.lzmafile.write(data) + + def close(self) -> None: + self.lzmafile.close() + self.file.close() + + +# io.IOBase is subclassed here, so that exporters can use the PostProcessingManager +# instance as a file like writable object. This could be needed by some exporters +# such as CsvItemExporter which wraps the feed storage with io.TextIOWrapper. +class PostProcessingManager(IOBase): + """ + This will manage and use declared plugins to process data in a + pipeline-ish way. + :param plugins: all the declared plugins for the feed + :type plugins: list + :param file: final target file where the processed data will be written + :type file: file like object + """ + + def __init__(self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]) -> None: + self.plugins = self._load_plugins(plugins) + self.file = file + self.feed_options = feed_options + self.head_plugin = self._get_head_plugin() + + def write(self, data: bytes) -> int: + """ + Uses all the declared plugins to process data first, then writes + the processed data to target file. + :param data: data passed to be written to target file + :type data: bytes + :return: returns number of bytes written + :rtype: int + """ + return self.head_plugin.write(data) + + def tell(self) -> int: + return self.file.tell() + + def close(self) -> None: + """ + Close the target file along with all the plugins. + """ + self.head_plugin.close() + + def writable(self) -> bool: + return True + + def _load_plugins(self, plugins: List[Any]) -> List[Any]: + plugins = [load_object(plugin) for plugin in plugins] + return plugins + + def _get_head_plugin(self) -> Any: + prev = self.file + for plugin in self.plugins[::-1]: + prev = plugin(prev, self.feed_options) + return prev diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index bcdbaff24..739e6b958 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -8,6 +8,7 @@ from scrapy import signals from scrapy.mail import MailSender from scrapy.exceptions import NotConfigured + class StatsMailer: def __init__(self, stats, recipients, mail): diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 0c97e6999..b43c383fe 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,10 +1,16 @@ +import re import time -from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE +from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode +# Defined in the http.cookiejar module, but undocumented: +# https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527 +IPV4_RE = re.compile(r"\.\d+$", re.ASCII) + + class CookieJar: def __init__(self, policy=None, check_expired_frequency=10000): self.policy = policy or DefaultCookiePolicy() @@ -136,10 +142,6 @@ class WrappedRequest: """ return self.request.meta.get('is_unverifiable', False) - def get_origin_req_host(self): - return urlparse_cached(self.request).hostname - - # python3 uses attributes instead of methods @property def full_url(self): return self.get_full_url() @@ -158,7 +160,7 @@ class WrappedRequest: @property def origin_req_host(self): - return self.get_origin_req_host() + return urlparse_cached(self.request).hostname def has_header(self, name): return name in self.request.headers diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 498f1b052..7672dec00 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,22 +4,57 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +import inspect +from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union + from w3lib.url import safe_url_string +import scrapy +from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers +from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -from scrapy.http.common import obsolete_setter -from scrapy.utils.curl import curl_to_request_kwargs + + +RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") class Request(object_ref): + """Represents an HTTP request, which is usually generated in a Spider and + executed by the Downloader, thus generating a :class:`Response`. + """ - def __init__(self, url, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, flags=None, cb_kwargs=None): + attributes: Tuple[str, ...] = ( + "url", "callback", "method", "headers", "body", + "cookies", "meta", "encoding", "priority", + "dont_filter", "errback", "flags", "cb_kwargs", + ) + """A tuple of :class:`str` objects containing the name of all public + attributes of the class that are also keyword parameters of the + ``__init__`` method. + Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and + :func:`~scrapy.utils.request.request_from_dict`. + """ + + def __init__( + self, + url: str, + callback: Optional[Callable] = None, + method: str = "GET", + headers: Optional[dict] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]] = None, + meta: Optional[dict] = None, + encoding: str = "utf-8", + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict] = None, + ) -> None: self._encoding = encoding # this one has to be set first self.method = str(method).upper() self._set_url(url) @@ -44,23 +79,23 @@ class Request(object_ref): self.flags = [] if flags is None else list(flags) @property - def cb_kwargs(self): + def cb_kwargs(self) -> dict: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self): + def meta(self) -> dict: if self._meta is None: self._meta = {} return self._meta - def _get_url(self): + def _get_url(self) -> str: return self._url - def _set_url(self, url): + def _set_url(self, url: str) -> None: if not isinstance(url, str): - raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}') + raise TypeError(f"Request url must be str, got {type(url).__name__}") s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) @@ -74,42 +109,37 @@ class Request(object_ref): url = property(_get_url, obsolete_setter(_set_url, 'url')) - def _get_body(self): + def _get_body(self) -> bytes: return self._body - def _set_body(self, body): - if body is None: - self._body = b'' - else: - self._body = to_bytes(body, self.encoding) + def _set_body(self, body: Optional[Union[str, bytes]]) -> None: + self._body = b"" if body is None else to_bytes(body, self.encoding) body = property(_get_body, obsolete_setter(_set_body, 'body')) @property - def encoding(self): + def encoding(self) -> str: return self._encoding - def __str__(self): + def __str__(self) -> str: return f"<{self.method} {self.url}>" __repr__ = __str__ - def copy(self): - """Return a copy of this Request""" + def copy(self) -> "Request": return self.replace() - def replace(self, *args, **kwargs): - """Create a new Request with the same attributes except for those - given new values. - """ - for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', - 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: + def replace(self, *args, **kwargs) -> "Request": + """Create a new Request with the same attributes except for those given new values""" + for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) @classmethod - def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs): + def from_curl( + cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, **kwargs + ) -> RequestTypeVar: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same @@ -136,8 +166,43 @@ class Request(object_ref): To translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. - - """ + """ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) request_kwargs.update(kwargs) return cls(**request_kwargs) + + def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict: + """Return a dictionary containing the Request's data. + + Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. + + If a spider is given, this method will try to find out the name of the spider methods used as callback + and errback and include them in the output dict, raising an exception if they cannot be found. + """ + d = { + "url": self.url, # urls are safe (safe_string_url) + "callback": _find_method(spider, self.callback) if callable(self.callback) else self.callback, + "errback": _find_method(spider, self.errback) if callable(self.errback) else self.errback, + "headers": dict(self.headers), + } + for attr in self.attributes: + d.setdefault(attr, getattr(self, attr)) + if type(self) is not Request: + d["_class"] = self.__module__ + '.' + self.__class__.__name__ + return d + + +def _find_method(obj, func): + """Helper function for Request.to_dict""" + # Only instance methods contain ``__func__`` + if obj and hasattr(func, '__func__'): + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original function object because instance + # method objects are generated each time attribute is retrieved from instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name + raise ValueError(f"Function {func} is not an instance method in: {obj}") diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2815303a2..0c947565a 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,22 +5,28 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from urllib.parse import urljoin, urlencode +from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union +from urllib.parse import urljoin, urlencode, urlsplit, urlunsplit -import lxml.html +from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request +from scrapy.http.response.text import TextResponse from scrapy.utils.python import to_bytes, is_listlike from scrapy.utils.response import get_base_url +FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") + +FormdataType = Optional[Union[dict, List[Tuple[str, str]]]] + + class FormRequest(Request): valid_form_methods = ['GET', 'POST'] - def __init__(self, *args, **kwargs): - formdata = kwargs.pop('formdata', None) + def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None: if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' @@ -28,17 +34,27 @@ class FormRequest(Request): if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata - querystr = _urlencode(items, self.encoding) + form_query_str = _urlencode(items, self.encoding) if self.method == 'POST': self.headers.setdefault(b'Content-Type', b'application/x-www-form-urlencoded') - self._set_body(querystr) + self._set_body(form_query_str) else: - self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr) + self._set_url(urlunsplit(urlsplit(self.url)._replace(query=form_query_str))) @classmethod - def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, - clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs): - + def from_response( + cls: Type[FormRequestTypeVar], + response: TextResponse, + formname: Optional[str] = None, + formid: Optional[str] = None, + formnumber: Optional[int] = 0, + formdata: FormdataType = None, + clickdata: Optional[dict] = None, + dont_click: bool = False, + formxpath: Optional[str] = None, + formcss: Optional[str] = None, + **kwargs, + ) -> FormRequestTypeVar: kwargs.setdefault('encoding', response.encoding) if formcss is not None: @@ -46,7 +62,7 @@ class FormRequest(Request): formxpath = HTMLTranslator().css_to_xpath(formcss) form = _get_form(response, formname, formid, formnumber, formxpath) - formdata = _get_inputs(form, formdata, dont_click, clickdata, response) + formdata = _get_inputs(form, formdata, dont_click, clickdata) url = _get_form_url(form, kwargs.pop('url', None)) method = kwargs.pop('method', form.method) @@ -58,7 +74,7 @@ class FormRequest(Request): return cls(url=url, method=method, formdata=formdata, **kwargs) -def _get_form_url(form, url): +def _get_form_url(form: FormElement, url: Optional[str]) -> str: if url is None: action = form.get('action') if action is None: @@ -67,17 +83,22 @@ def _get_form_url(form, url): return urljoin(form.base_url, url) -def _urlencode(seq, enc): +def _urlencode(seq: Iterable, enc: str) -> str: values = [(to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq for v in (vs if is_listlike(vs) else [vs])] - return urlencode(values, doseq=1) + return urlencode(values, doseq=True) -def _get_form(response, formname, formid, formnumber, formxpath): - """Find the form element """ - root = create_root_node(response.text, lxml.html.HTMLParser, - base_url=get_base_url(response)) +def _get_form( + response: TextResponse, + formname: Optional[str], + formid: Optional[str], + formnumber: Optional[int], + formxpath: Optional[str], +) -> FormElement: + """Find the wanted form element within the given response.""" + root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: raise ValueError(f"No
element found in {response}") @@ -105,8 +126,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): break raise ValueError(f'No element found with {formxpath}') - # If we get here, it means that either formname was None - # or invalid + # If we get here, it means that either formname was None or invalid if formnumber is not None: try: form = forms[formnumber] @@ -116,25 +136,32 @@ def _get_form(response, formname, formid, formnumber, formxpath): return form -def _get_inputs(form, formdata, dont_click, clickdata, response): +def _get_inputs( + form: FormElement, + formdata: FormdataType, + dont_click: bool, + clickdata: Optional[dict], +) -> List[Tuple[str, str]]: + """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): raise ValueError('formdata should be a dict or iterable of tuples') if not formdata: - formdata = () + formdata = [] inputs = form.xpath('descendant::textarea' '|descendant::select' '|descendant::input[not(@type) or @type[' ' not(re:test(., "^(?:submit|image|reset)$", "i"))' ' and (../@checked or' ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', - namespaces={ - "re": "http://exslt.org/regular-expressions"}) - values = [(k, '' if v is None else v) - for k, v in (_value(e) for e in inputs) - if k and k not in formdata_keys] + namespaces={"re": "http://exslt.org/regular-expressions"}) + values = [ + (k, '' if v is None else v) + for k, v in (_value(e) for e in inputs) + if k and k not in formdata_keys + ] if not dont_click: clickable = _get_clickable(clickdata, form) @@ -142,13 +169,13 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): values.append(clickable) if isinstance(formdata, dict): - formdata = formdata.items() + formdata = formdata.items() # type: ignore[assignment] values.extend((k, v) for k, v in formdata if v is not None) return values -def _value(ele): +def _value(ele: HtmlElement): n = ele.name v = ele.value if ele.tag == 'select': @@ -156,22 +183,23 @@ def _value(ele): return n, v -def _select_value(ele, n, v): +def _select_value(ele: SelectElement, n: str, v: str): multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected - # And for select tags wihout options + # And for select tags without options o = ele.value_options return (n, o[0]) if o else (None, None) elif v is not None and multiple: # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') - v = [(o.get('value') or o.text or '').strip() for o in selected_options] + values = [(o.get('value') or o.text or '').strip() for o in selected_options] + return n, values return n, v -def _get_clickable(clickdata, form): +def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tuple[str, str]]: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first @@ -183,7 +211,7 @@ def _get_clickable(clickdata, form): namespaces={"re": "http://exslt.org/regular-expressions"} )) if not clickables: - return + return None # If we don't have clickdata, we just use the first clickable element if clickdata is None: diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index eae3f9f6b..728a2a104 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,14 +8,18 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings +from typing import Optional, Tuple from scrapy.http.request import Request from scrapy.utils.deprecate import create_deprecated_class class JsonRequest(Request): - def __init__(self, *args, **kwargs): - dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) + + attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) + + def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: + dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault('sort_keys', True) self._dumps_kwargs = dumps_kwargs @@ -25,10 +29,8 @@ class JsonRequest(Request): if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) - if 'method' not in kwargs: kwargs['method'] = 'POST' @@ -36,20 +38,23 @@ class JsonRequest(Request): self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') - def replace(self, *args, **kwargs): + @property + def dumps_kwargs(self) -> dict: + return self._dumps_kwargs + + def replace(self, *args, **kwargs) -> Request: body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) return super().replace(*args, **kwargs) - def _dumps(self, data): + def _dumps(self, data: dict) -> str: """Convert to JSON """ return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index c70912e49..06d98cea5 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -5,6 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ import xmlrpc.client as xmlrpclib +from typing import Optional from scrapy.http.request import Request from scrapy.utils.python import get_func_args @@ -15,8 +16,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args, **kwargs): - encoding = kwargs.get('encoding', None) + def __init__(self, *args, encoding: Optional[str] = None, **kwargs): if 'body' not in kwargs and 'params' in kwargs: kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) kwargs['body'] = xmlrpclib.dumps(**kw) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c635fde69..4de6c9b5b 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,7 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -from typing import Generator +from typing import Generator, Tuple from urllib.parse import urljoin from scrapy.exceptions import NotSupported @@ -16,9 +16,32 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): + """An object that represents an HTTP response, which is usually + downloaded (by the Downloader) and fed to the Spiders for processing. + """ - def __init__(self, url, status=200, headers=None, body=b'', flags=None, - request=None, certificate=None, ip_address=None): + attributes: Tuple[str, ...] = ( + "url", "status", "headers", "body", "flags", "request", "certificate", "ip_address", "protocol", + ) + """A tuple of :class:`str` objects containing the name of all public + attributes of the class that are also keyword parameters of the + ``__init__`` method. + + Currently used by :meth:`Response.replace`. + """ + + def __init__( + self, + url, + status=200, + headers=None, + body=b"", + flags=None, + request=None, + certificate=None, + ip_address=None, + protocol=None, + ): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) @@ -27,6 +50,7 @@ class Response(object_ref): self.flags = [] if flags is None else list(flags) self.certificate = certificate self.ip_address = ip_address + self.protocol = protocol @property def cb_kwargs(self): @@ -86,11 +110,8 @@ class Response(object_ref): return self.replace() def replace(self, *args, **kwargs): - """Create a new Response with the same attributes except for those - given new values. - """ - for x in ['url', 'status', 'headers', 'body', - 'request', 'flags', 'certificate', 'ip_address']: + """Create a new Response with the same attributes except for those given new values""" + for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index e36e14880..89516b9b6 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -6,9 +6,8 @@ See documentation in docs/topics/request-response.rst """ import json -import warnings from contextlib import suppress -from typing import Generator +from typing import Generator, Tuple from urllib.parse import urljoin import parsel @@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, http_content_type_encoding, resolve_encoding) from w3lib.html import strip_html5_whitespace -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode @@ -30,6 +28,8 @@ class TextResponse(Response): _DEFAULT_ENCODING = 'ascii' _cached_decoded_json = _NONE + attributes: Tuple[str, ...] = Response.attributes + ("encoding",) + def __init__(self, *args, **kwargs): self._encoding = kwargs.pop('encoding', None) self._cached_benc = None @@ -53,10 +53,6 @@ class TextResponse(Response): else: super()._set_body(body) - def replace(self, *args, **kwargs): - kwargs.setdefault('encoding', self.encoding) - return Response.replace(self, *args, **kwargs) - @property def encoding(self): return self._declared_encoding() or self._body_inferred_encoding() @@ -68,13 +64,6 @@ class TextResponse(Response): or self._body_declared_encoding() ) - def body_as_unicode(self): - """Return body as unicode""" - warnings.warn('Response.body_as_unicode() is deprecated, ' - 'please use Response.text instead.', - ScrapyDeprecationWarning, stacklevel=2) - return self.text - def json(self): """ .. versionadded:: 2.2 diff --git a/scrapy/item.py b/scrapy/item.py index af3849302..2521ac829 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -8,45 +8,16 @@ from abc import ABCMeta from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat -from warnings import warn +from typing import Dict -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref -class _BaseItem(object_ref): - """ - Temporary class used internally to avoid the deprecation - warning raised by isinstance checks using BaseItem. - """ - pass - - -class _BaseItemMeta(ABCMeta): - def __instancecheck__(cls, instance): - if cls is BaseItem: - warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__instancecheck__(instance) - - -class BaseItem(_BaseItem, metaclass=_BaseItemMeta): - """ - Deprecated, please use :class:`scrapy.item.Item` instead - """ - - def __new__(cls, *args, **kwargs): - if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): - warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) - - class Field(dict): """Container of field metadata""" -class ItemMeta(_BaseItemMeta): +class ItemMeta(ABCMeta): """Metaclass_ of :class:`Item` that handles field definitions. .. _metaclass: https://realpython.com/python-metaclasses @@ -73,15 +44,30 @@ class ItemMeta(_BaseItemMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class DictItem(MutableMapping, BaseItem): +class Item(MutableMapping, object_ref, metaclass=ItemMeta): + """ + Base class for scraped items. - fields = {} + In Scrapy, an object is considered an ``item`` if it is an instance of either + :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a + spider callback is evaluated, only instances of :class:`Item` or + :class:`dict` are passed to :ref:`item pipelines `. - def __new__(cls, *args, **kwargs): - if issubclass(cls, DictItem) and not issubclass(cls, Item): - warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) + If you need instances of a custom class to be considered items by Scrapy, + you must inherit from either :class:`Item` or :class:`dict`. + + Items must declare :class:`Field` attributes, which are processed and stored + in the ``fields`` attribute. This restricts the set of allowed field names + and prevents typos, raising ``KeyError`` when referring to undefined fields. + Additionally, fields can be used to define metadata and control the way + data is processed internally. Please refer to the :ref:`documentation + about fields ` for additional information. + + Unlike instances of :class:`dict`, instances of :class:`Item` may be + :ref:`tracked ` to debug memory leaks. + """ + + fields: Dict[str, Field] def __init__(self, *args, **kwargs): self._values = {} @@ -117,7 +103,7 @@ class DictItem(MutableMapping, BaseItem): def __iter__(self): return iter(self._values) - __hash__ = BaseItem.__hash__ + __hash__ = object_ref.__hash__ def keys(self): return self._values.keys() @@ -132,27 +118,3 @@ class DictItem(MutableMapping, BaseItem): """Return a :func:`~copy.deepcopy` of this item. """ return deepcopy(self) - - -class Item(DictItem, metaclass=ItemMeta): - """ - Base class for scraped items. - - In Scrapy, an object is considered an ``item`` if it is an instance of either - :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a - spider callback is evaluated, only instances of :class:`Item` or - :class:`dict` are passed to :ref:`item pipelines `. - - If you need instances of a custom class to be considered items by Scrapy, - you must inherit from either :class:`Item` or :class:`dict`. - - Items must declare :class:`Field` attributes, which are processed and stored - in the ``fields`` attribute. This restricts the set of allowed field names - and prevents typos, raising ``KeyError`` when referring to undefined fields. - Additionally, fields can be used to define metadata and control the way - data is processed internally. Please refer to the :ref:`documentation - about fields ` for additional information. - - Unlike instances of :class:`dict`, instances of :class:`Item` may be - :ref:`tracked ` to debug memory leaks. - """ diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index e941c4321..b5d2585a8 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -88,7 +88,7 @@ class LxmlParserLinkExtractor: def _process_links(self, links): """ Normalize and filter extracted links - The subclass should override it if neccessary + The subclass should override it if necessary """ return self._deduplicate_if_needed(links) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 014951a8e..91337b949 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -83,6 +83,9 @@ class ItemLoader(itemloaders.ItemLoader): def __init__(self, item=None, selector=None, response=None, parent=None, **context): if selector is None and response is not None: - selector = self.default_selector_class(response) + try: + selector = self.default_selector_class(response) + except AttributeError: + selector = None context.update(response=response) super().__init__(item=item, selector=selector, parent=parent, **context) diff --git a/scrapy/mail.py b/scrapy/mail.py index 7d7a2c435..2a25ccd44 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -9,7 +9,7 @@ from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText -from email.utils import COMMASPACE, formatdate +from email.utils import formatdate from io import BytesIO from twisted.internet import defer, ssl @@ -21,6 +21,11 @@ from scrapy.utils.python import to_bytes logger = logging.getLogger(__name__) +# Defined in the email.utils module, but undocumented: +# https://github.com/python/cpython/blob/v3.9.0/Lib/email/utils.py#L42 +COMMASPACE = ", " + + def _to_bytes_or_none(text): if text is None: return None diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 5040378ea..2eb1d8609 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,10 +1,15 @@ -from collections import defaultdict, deque import logging import pprint +from collections import defaultdict, deque +from typing import Callable, Deque, Dict, Optional, cast, Iterable +from twisted.internet.defer import Deferred + +from scrapy import Spider from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.defer import process_parallel, process_chain, process_chain_both +from scrapy.utils.defer import process_parallel, process_chain logger = logging.getLogger(__name__) @@ -16,16 +21,17 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods = defaultdict(deque) + # Optional because process_spider_output and process_spider_exception can be None + self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: Settings) -> list: raise NotImplementedError @classmethod - def from_settings(cls, settings, crawler=None): + def from_settings(cls, settings: Settings, crawler=None): mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -52,24 +58,22 @@ class MiddlewareManager: def from_crawler(cls, crawler): return cls.from_settings(crawler.settings, crawler) - def _add_middleware(self, mw): + def _add_middleware(self, mw) -> None: if hasattr(mw, 'open_spider'): self.methods['open_spider'].append(mw.open_spider) if hasattr(mw, 'close_spider'): self.methods['close_spider'].appendleft(mw.close_spider) - def _process_parallel(self, methodname, obj, *args): - return process_parallel(self.methods[methodname], obj, *args) + def _process_parallel(self, methodname: str, obj, *args) -> Deferred: + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_parallel(methods, obj, *args) - def _process_chain(self, methodname, obj, *args): - return process_chain(self.methods[methodname], obj, *args) + def _process_chain(self, methodname: str, obj, *args) -> Deferred: + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_chain(methods, obj, *args) - def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args): - return process_chain_both(self.methods[cb_methodname], - self.methods[eb_methodname], obj, *args) - - def open_spider(self, spider): + def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) - def close_spider(self, spider): + def close_spider(self, spider: Spider) -> Deferred: return self._process_parallel('close_spider', spider) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 13ecd4e6c..906e7eb24 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -79,12 +79,13 @@ class FSFilesStore: class S3FilesStore: AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None + AWS_SESSION_TOKEN = None AWS_ENDPOINT_URL = None AWS_REGION_NAME = None AWS_USE_SSL = None AWS_VERIFY = None - POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings + POLICY = 'private' # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings HEADERS = { 'Cache-Control': 'max-age=172800', } @@ -98,6 +99,7 @@ class S3FilesStore: 's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID, aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + aws_session_token=self.AWS_SESSION_TOKEN, endpoint_url=self.AWS_ENDPOINT_URL, region_name=self.AWS_REGION_NAME, use_ssl=self.AWS_USE_SSL, @@ -140,7 +142,7 @@ class S3FilesStore: **extra) def _headers_to_botocore_kwargs(self, headers): - """ Convert headers to botocore keyword agruments. + """ Convert headers to botocore keyword arguments. """ # This is required while we need to support both boto and botocore. mapping = CaselessDict({ @@ -188,7 +190,7 @@ class GCSFilesStore: CACHE_CONTROL = 'max-age=172800' # The bucket's default object ACL will be applied to the object. - # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. POLICY = None def __init__(self, uri): @@ -220,8 +222,8 @@ class GCSFilesStore: return {'checksum': checksum, 'last_modified': last_modified} else: return {} - - return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + blob_path = self._get_blob_path(path) + return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess) def _get_content_type(self, headers): if headers and 'Content-Type' in headers: @@ -229,8 +231,12 @@ class GCSFilesStore: else: return 'application/octet-stream' + def _get_blob_path(self, path): + return self.prefix + path + def persist_file(self, path, buf, info, meta=None, headers=None): - blob = self.bucket.blob(self.prefix + path) + blob_path = self._get_blob_path(path) + blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL blob.metadata = {k: str(v) for k, v in (meta or {}).items()} return threads.deferToThread( @@ -289,7 +295,7 @@ class FilesPipeline(MediaPipeline): """Abstract pipeline that implement the file downloading This pipeline tries to minimize network transfers and file processing, - doing stat of the files and determining if file is new, uptodate or + doing stat of the files and determining if file is new, up-to-date or expired. ``new`` files are those that pipeline never processed and needs to be @@ -349,6 +355,7 @@ class FilesPipeline(MediaPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index aafd1d8b2..6b97190ee 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -9,9 +9,8 @@ from contextlib import suppress from io import BytesIO from itemadapter import ItemAdapter -from PIL import Image -from scrapy.exceptions import DropItem +from scrapy.exceptions import DropItem, NotConfigured from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline @@ -45,6 +44,14 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): + try: + from PIL import Image + self._Image = Image + except ImportError: + raise NotConfigured( + 'ImagesPipeline requires installing Pillow 4.0.0 or later' + ) + super().__init__(store_uri, settings=settings, download_func=download_func) if isinstance(settings, dict) or settings is None: @@ -85,6 +92,7 @@ class ImagesPipeline(FilesPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] @@ -121,7 +129,7 @@ class ImagesPipeline(FilesPipeline): def get_images(self, response, request, info, *, item=None): path = self.file_path(request, response=response, info=info, item=item) - orig_image = Image.open(BytesIO(response.body)) + orig_image = self._Image.open(BytesIO(response.body)) width, height = orig_image.size if width < self.min_width or height < self.min_height: @@ -133,18 +141,18 @@ class ImagesPipeline(FilesPipeline): yield path, image, buf for thumb_id, size in self.thumbs.items(): - thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) + thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item) thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf def convert_image(self, image, size=None): if image.format == 'PNG' and image.mode == 'RGBA': - background = Image.new('RGBA', image.size, (255, 255, 255)) + background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') elif image.mode == 'P': image = image.convert("RGBA") - background = Image.new('RGBA', image.size, (255, 255, 255)) + background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') elif image.mode != 'RGB': @@ -152,7 +160,7 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() - image.thumbnail(size, Image.ANTIALIAS) + image.thumbnail(size, self._Image.ANTIALIAS) buf = BytesIO() image.save(buf, 'JPEG') @@ -171,6 +179,6 @@ class ImagesPipeline(FilesPipeline): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return f'full/{image_guid}.jpg' - def thumb_path(self, request, thumb_id, response=None, info=None): + def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return f'thumbs/{thumb_id}/{thumb_guid}.jpg' diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0a12f3e2c..5308a9793 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -11,7 +11,6 @@ from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter from scrapy.utils.log import failure_to_exc_info @@ -77,6 +76,7 @@ class MediaPipeline: except AttributeError: pipe = cls() pipe.crawler = crawler + pipe._fingerprinter = crawler.request_fingerprinter return pipe def open_spider(self, spider): @@ -86,11 +86,11 @@ class MediaPipeline: info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] - dfd = DeferredList(dlist, consumeErrors=1) + dfd = DeferredList(dlist, consumeErrors=True) return dfd.addCallback(self.item_completed, item, info) def _process_request(self, request, info, item): - fp = request_fingerprint(request) + fp = self._fingerprinter.fingerprint(request) cb = request.callback or (lambda _: _) eb = request.errback request.callback = None @@ -121,7 +121,7 @@ class MediaPipeline: def _make_compatible(self): """Make overridable methods of MediaPipeline and subclasses backwards compatible""" methods = [ - "file_path", "media_to_download", "media_downloaded", + "file_path", "thumb_path", "media_to_download", "media_downloaded", "file_downloaded", "image_downloaded", "get_images" ] @@ -173,7 +173,7 @@ class MediaPipeline: errback=self.media_failed, errbackArgs=(request, info)) else: self._modify_media_request(request) - dfd = self.crawler.engine.download(request, info.spider) + dfd = self.crawler.engine.download(request) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index a9aa6c649..b4b63e7c7 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -3,6 +3,7 @@ import logging from scrapy.utils.misc import create_instance + logger = logging.getLogger(__name__) @@ -17,8 +18,7 @@ def _path_safe(text): >>> _path_safe('some@symbol?').startswith('some_symbol_') True """ - pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' - for c in text]) + pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part unique_slot = hashlib.md5(text.encode('utf8')).hexdigest() @@ -35,6 +35,9 @@ class ScrapyPriorityQueue: * close() * __len__() + Optionally, the queue could provide a ``peek`` method, that should return the + next object to be returned by ``pop``, but without removing it from the queue. + ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls argument, which is a class used to instantiate a new (internal) queue when a new priority is allocated. @@ -70,10 +73,12 @@ class ScrapyPriorityQueue: self.curprio = min(startprios) def qfactory(self, key): - return create_instance(self.downstream_queue_cls, - None, - self.crawler, - self.key + '/' + str(key)) + return create_instance( + self.downstream_queue_cls, + None, + self.crawler, + self.key + '/' + str(key), + ) def priority(self, request): return -request.priority @@ -99,6 +104,18 @@ class ScrapyPriorityQueue: self.curprio = min(prios) if prios else None return m + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + if self.curprio is None: + return None + queue = self.queues[self.curprio] + return queue.peek() + def close(self): active = [] for p, q in self.queues.items(): @@ -116,8 +133,7 @@ class DownloaderInterface: self.downloader = crawler.engine.downloader def stats(self, possible_slots): - return [(self._active_downloads(slot), slot) - for slot in possible_slots] + return [(self._active_downloads(slot), slot) for slot in possible_slots] def get_slot_key(self, request): return self.downloader._get_slot_key(request, None) @@ -162,10 +178,12 @@ class DownloaderAwarePriorityQueue: self.pqueues[slot] = self.pqfactory(slot, startprios) def pqfactory(self, slot, startprios=()): - return ScrapyPriorityQueue(self.crawler, - self.downstream_queue_cls, - self.key + '/' + _path_safe(slot), - startprios) + return ScrapyPriorityQueue( + self.crawler, + self.downstream_queue_cls, + self.key + '/' + _path_safe(slot), + startprios, + ) def pop(self): stats = self._downloader_interface.stats(self.pqueues) @@ -187,9 +205,22 @@ class DownloaderAwarePriorityQueue: queue = self.pqueues[slot] queue.push(request) + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + stats = self._downloader_interface.stats(self.pqueues) + if not stats: + return None + slot = min(stats)[1] + queue = self.pqueues[slot] + return queue.peek() + def close(self): - active = {slot: queue.close() - for slot, queue in self.pqueues.items()} + active = {slot: queue.close() for slot, queue in self.pqueues.items()} self.pqueues.clear() return active diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index a25871433..08f08e8d7 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -69,7 +69,7 @@ class Selector(_ParselSelector, object_ref): raise ValueError(f'{self.__class__.__name__}.__init__() received ' 'both response and text') - st = _st(response, type or self._default_type) + st = _st(response, type) if text is not None: response = _response_from_text(text, st) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 53c556ebb..2bbe38481 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -408,9 +408,13 @@ class BaseSettings(MutableMapping): return len(self.attributes) def _to_dict(self): - return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) + return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items()} + def _get_key(self, key_value): + return (key_value if isinstance(key_value, (bool, float, int, str, type(None))) + else str(key_value)) + def copy_to_dict(self): """ Make a copy of current settings and convert to a dict. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4ef330dd2..f5a3efe69 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -207,6 +207,7 @@ LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' LOG_FILE = None +LOG_FILE_APPEND = True LOG_SHORT_NAMES = False SCHEDULER_DEBUG = False @@ -245,6 +246,9 @@ REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' +REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter' +REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION' + RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] diff --git a/scrapy/shell.py b/scrapy/shell.py index c370ccaff..f2dff2ae3 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -79,7 +79,7 @@ class Shell: spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) - self.crawler.engine.crawl(request, spider) + self.crawler.engine.crawl(request) return d def _open_spider(self, request, spider): diff --git a/scrapy/signals.py b/scrapy/signals.py index c61ae6ec3..8cf2a4d93 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -17,6 +17,7 @@ request_reached_downloader = object() request_left_downloader = object() response_received = object() response_downloaded = object() +headers_received = object() bytes_received = object() item_scraped = object() item_dropped = object() diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index ae5c258df..9861456de 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -32,7 +32,7 @@ class HttpErrorMiddleware: if 200 <= response.status < 300: # common case return meta = response.meta - if 'handle_httpstatus_all' in meta: + if meta.get('handle_httpstatus_all', False): return if 'handle_httpstatus_list' in meta: allowed_statuses = meta['handle_httpstatus_list'] diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f81041376..608c0eea5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -3,15 +3,16 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ import warnings +from typing import Tuple from urllib.parse import urlparse from w3lib.url import safe_url_string -from scrapy.http import Request, Response -from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.python import to_unicode +from scrapy.exceptions import NotConfigured +from scrapy.http import Request, Response from scrapy.utils.misc import load_object +from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -30,7 +31,8 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default" class ReferrerPolicy: - NOREFERRER_SCHEMES = LOCAL_SCHEMES + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + name: str def referrer(self, response_url, request_url): raise NotImplementedError() @@ -88,7 +90,7 @@ class NoReferrerPolicy(ReferrerPolicy): is to be sent along with requests made from a particular request client to any origin. The header will be omitted entirely. """ - name = POLICY_NO_REFERRER + name: str = POLICY_NO_REFERRER def referrer(self, response_url, request_url): return None @@ -108,7 +110,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): This is a user agent's default behavior, if no policy is otherwise specified. """ - name = POLICY_NO_REFERRER_WHEN_DOWNGRADE + name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response_url, request_url): if not self.tls_protected(response_url) or self.tls_protected(request_url): @@ -125,7 +127,7 @@ class SameOriginPolicy(ReferrerPolicy): Cross-origin requests, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_SAME_ORIGIN + name: str = POLICY_SAME_ORIGIN def referrer(self, response_url, request_url): if self.origin(response_url) == self.origin(request_url): @@ -141,7 +143,7 @@ class OriginPolicy(ReferrerPolicy): when making both same-origin requests and cross-origin requests from a particular request client. """ - name = POLICY_ORIGIN + name: str = POLICY_ORIGIN def referrer(self, response_url, request_url): return self.origin_referrer(response_url) @@ -160,7 +162,7 @@ class StrictOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_STRICT_ORIGIN + name: str = POLICY_STRICT_ORIGIN def referrer(self, response_url, request_url): if ( @@ -181,7 +183,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): is sent as referrer information when making cross-origin requests from a particular request client. """ - name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN + name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): origin = self.origin(response_url) @@ -208,7 +210,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN + name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response_url, request_url): origin = self.origin(response_url) @@ -234,7 +236,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): to insecure origins. Carefully consider the impact of setting such a policy for potentially sensitive documents. """ - name = POLICY_UNSAFE_URL + name: str = POLICY_UNSAFE_URL def referrer(self, response_url, request_url): return self.stripped_referrer(response_url) @@ -246,8 +248,8 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): with the addition that "Referer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. """ - NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') - name = POLICY_SCRAPY_DEFAULT + NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ('file', 's3') + name: str = POLICY_SCRAPY_DEFAULT _policy_classes = {p.name: p for p in ( diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 5be1f80cb..450d4ff40 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -27,9 +27,12 @@ class UrlLengthMiddleware: def process_spider_output(self, response, result, spider): def _filter(request): if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.debug("Ignoring link (url length > %(maxlength)d): %(url)s ", - {'maxlength': self.maxlength, 'url': request.url}, - extra={'spider': spider}) + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {'maxlength': self.maxlength, 'url': request.url}, + extra={'spider': spider} + ) + spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider) return False else: return True diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c13ba4b3c..d8248c606 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -4,14 +4,12 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ import logging -import warnings from typing import Optional from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider -from scrapy.utils.deprecate import method_is_overridden class Spider(object_ref): @@ -57,34 +55,13 @@ class Spider(object_ref): crawler.signals.connect(self.close, signals.spider_closed) def start_requests(self): - cls = self.__class__ if not self.start_urls and hasattr(self, 'start_url'): raise AttributeError( "Crawling could not start: 'start_urls' not found " "or empty (but found 'start_url' attribute instead, " "did you miss an 's'?)") - if method_is_overridden(cls, Spider, 'make_requests_from_url'): - warnings.warn( - "Spider.make_requests_from_url method is deprecated; it " - "won't be called in future Scrapy releases. Please " - "override Spider.start_requests method instead " - f"(see {cls.__module__}.{cls.__name__}).", - ) - for url in self.start_urls: - yield self.make_requests_from_url(url) - else: - for url in self.start_urls: - yield Request(url, dont_filter=True) - - def make_requests_from_url(self, url): - """ This method is deprecated. """ - warnings.warn( - "Spider.make_requests_from_url method is deprecated: " - "it will be removed and not be called by the default " - "Spider.start_requests method in future Scrapy releases. " - "Please override Spider.start_requests method instead." - ) - return Request(url, dont_filter=True) + for url in self.start_urls: + yield Request(url, dont_filter=True) def _parse(self, response, **kwargs): return self.parse(response, **kwargs) diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6ed17e4dd..79e12e030 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -43,7 +43,7 @@ class XMLFeedSpider(Spider): return response def parse_node(self, response, selector): - """This method must be overriden with your custom spider functionality""" + """This method must be overridden with your custom spider functionality""" if hasattr(self, 'parse_item'): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError @@ -113,7 +113,7 @@ class CSVFeedSpider(Spider): return response def parse_row(self, response, row): - """This method must be overriden with your custom spider functionality""" + """This method must be overridden with your custom spider functionality""" raise NotImplementedError def parse_rows(self, response): @@ -123,7 +123,7 @@ class CSVFeedSpider(Spider): process_results methods for pre and post-processing purposes. """ - for row in csviter(response, self.delimiter, self.headers, self.quotechar): + for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar): ret = iterate_spider_output(self.parse_row(response, row)) for result_item in self.process_results(response, ret): yield result_item diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 77ffda6f7..dff9b1350 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -8,7 +8,8 @@ import pickle from queuelib import queue -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy.utils.deprecate import create_deprecated_class +from scrapy.utils.request import request_from_dict def _with_mkdir(queue_class): @@ -19,7 +20,6 @@ def _with_mkdir(queue_class): dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, exist_ok=True) - super().__init__(path, *args, **kwargs) return DirectoriesCreated @@ -38,6 +38,20 @@ def _serializable_queue(queue_class, serialize, deserialize): if s: return deserialize(s) + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + try: + s = super().peek() + except AttributeError as ex: + raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex + if s: + return deserialize(s) + return SerializableQueue @@ -54,17 +68,26 @@ def _scrapy_serialization_queue(queue_class): return cls(crawler, key) def push(self, request): - request = request_to_dict(request, self.spider) + request = request.to_dict(spider=self.spider) return super().push(request) def pop(self): request = super().pop() - if not request: return None + return request_from_dict(request, spider=self.spider) - request = request_from_dict(request, self.spider) - return request + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + request = super().peek() + if not request: + return None + return request_from_dict(request, spider=self.spider) return ScrapyRequestQueue @@ -76,6 +99,19 @@ def _scrapy_non_serialization_queue(queue_class): def from_crawler(cls, crawler, *args, **kwargs): return cls() + def peek(self): + """Returns the next object to be returned by :meth:`pop`, + but without removing it from the queue. + + Raises :exc:`NotImplementedError` if the underlying queue class does + not implement a ``peek`` method, which is optional for queues. + """ + try: + s = super().peek() + except AttributeError as ex: + raise NotImplementedError("The underlying queue class does not implement 'peek'") from ex + return s + return ScrapyRequestQueue @@ -88,38 +124,60 @@ def _pickle_serialize(obj): raise ValueError(str(e)) from e -PickleFifoDiskQueueNonRequest = _serializable_queue( +_PickleFifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads ) -PickleLifoDiskQueueNonRequest = _serializable_queue( +_PickleLifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads ) -MarshalFifoDiskQueueNonRequest = _serializable_queue( +_MarshalFifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads ) -MarshalLifoDiskQueueNonRequest = _serializable_queue( +_MarshalLifoSerializationDiskQueue = _serializable_queue( _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads ) -PickleFifoDiskQueue = _scrapy_serialization_queue( - PickleFifoDiskQueueNonRequest -) -PickleLifoDiskQueue = _scrapy_serialization_queue( - PickleLifoDiskQueueNonRequest -) -MarshalFifoDiskQueue = _scrapy_serialization_queue( - MarshalFifoDiskQueueNonRequest -) -MarshalLifoDiskQueue = _scrapy_serialization_queue( - MarshalLifoDiskQueueNonRequest -) +# public queue classes +PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue) +PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue) +MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue) +MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) + + +# deprecated queue classes +_subclass_warn_message = "{cls} inherits from deprecated class {old}" +_instance_warn_message = "{cls} is deprecated" +PickleFifoDiskQueueNonRequest = create_deprecated_class( + name="PickleFifoDiskQueueNonRequest", + new_class=_PickleFifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +PickleLifoDiskQueueNonRequest = create_deprecated_class( + name="PickleLifoDiskQueueNonRequest", + new_class=_PickleLifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +MarshalFifoDiskQueueNonRequest = create_deprecated_class( + name="MarshalFifoDiskQueueNonRequest", + new_class=_MarshalFifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) +MarshalLifoDiskQueueNonRequest = create_deprecated_class( + name="MarshalLifoDiskQueueNonRequest", + new_class=_MarshalLifoSerializationDiskQueue, + subclass_warn_message=_subclass_warn_message, + instance_warn_message=_instance_warn_message, +) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index a414b5fde..5e541e2c0 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -86,3 +86,6 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_DIR = 'httpcache' #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' + +# Set settings whose default value is deprecated to a future-proof value +REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION' diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py new file mode 100644 index 000000000..c290e376c --- /dev/null +++ b/scrapy/utils/asyncgen.py @@ -0,0 +1,8 @@ +from collections.abc import AsyncIterable + + +async def collect_asyncgen(result: AsyncIterable): + results = [] + async for x in result: + results.append(x) + return results diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 453d8b487..00cc53725 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -121,7 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings): out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) - out.setdefault("item_export_kwargs", dict()) + out.setdefault("item_export_kwargs", {}) if settings["FEED_EXPORT_INDENT"] is None: out.setdefault("indent", None) else: @@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None, message = ( 'The -t command line option is deprecated in favor of ' 'specifying the output format within the output URI. See the ' - 'documentation of the -o and -O options for more information.', + 'documentation of the -o and -O options for more information.' ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 133261fd7..1bc0bd45f 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -14,7 +14,7 @@ def _embed_ipython_shell(namespace={}, banner=''): @wraps(_embed_ipython_shell) def wrapper(namespace=namespace, banner=''): config = load_default_config() - # Always use .instace() to ensure _instance propagation to all parents + # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports # and clear the instance to always have the fresh env # on repeated breaks like with inspect_response() diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 6660b9dc0..74f82ad75 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -34,7 +34,27 @@ for argument in safe_to_ignore_arguments: curl_parser.add_argument(*argument, action='store_true') -def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): +def _parse_headers_and_cookies(parsed_args): + headers = [] + cookies = {} + for header in parsed_args.headers or (): + name, val = header.split(':', 1) + name = name.strip() + val = val.strip() + if name.title() == 'Cookie': + for name, morsel in SimpleCookie(val).items(): + cookies[name] = morsel.value + else: + headers.append((name, val)) + + if parsed_args.auth: + user, password = parsed_args.auth.split(':', 1) + headers.append(('Authorization', basic_auth_header(user, password))) + + return headers, cookies + + +def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = True) -> dict: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command @@ -70,21 +90,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): result = {'method': method.upper(), 'url': url} - headers = [] - cookies = {} - for header in parsed_args.headers or (): - name, val = header.split(':', 1) - name = name.strip() - val = val.strip() - if name.title() == 'Cookie': - for name, morsel in SimpleCookie(val).items(): - cookies[name] = morsel.value - else: - headers.append((name, val)) - - if parsed_args.auth: - user, password = parsed_args.auth.split(':', 1) - headers.append(('Authorization', basic_auth_header(user, password))) + headers, cookies = _parse_headers_and_cookies(parsed_args) if headers: result['headers'] = headers diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index e31284a7f..47df8a717 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -41,7 +41,7 @@ class CaselessDict(dict): return key.lower() def normvalue(self, value): - """Method to normalize values prior to be setted""" + """Method to normalize values prior to be set""" return value def get(self, key, def_val=None): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 21ba02a0b..7ecb8ea3f 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -3,16 +3,28 @@ Helper functions for dealing with Twisted deferreds """ import asyncio import inspect +from asyncio import Future from functools import wraps +from typing import ( + Any, + Callable, + Coroutine, + Generator, + Iterable, + Union +) -from twisted.internet import defer, task +from twisted.internet import defer +from twisted.internet.defer import Deferred, DeferredList, ensureDeferred +from twisted.internet.task import Cooperator from twisted.python import failure +from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest from scrapy.utils.reactor import is_asyncio_reactor_installed -def defer_fail(_failure): +def defer_fail(_failure: Failure) -> Deferred: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -20,26 +32,26 @@ def defer_fail(_failure): before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.errback, _failure) return d -def defer_succeed(result): +def defer_succeed(result) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop - It delays by 100ms so reactor has a chance to go trough readers and writers + It delays by 100ms so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ from twisted.internet import reactor - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.callback, result) return d -def defer_result(result): - if isinstance(result, defer.Deferred): +def defer_result(result) -> Deferred: + if isinstance(result, Deferred): return result elif isinstance(result, failure.Failure): return defer_fail(result) @@ -47,7 +59,7 @@ def defer_result(result): return defer_succeed(result) -def mustbe_deferred(f, *args, **kw): +def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -64,29 +76,29 @@ def mustbe_deferred(f, *args, **kw): return defer_result(result) -def parallel(iterable, count, callable, *args, **named): +def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) -> DeferredList: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. Taken from: https://jcalderone.livejournal.com/24285.html """ - coop = task.Cooperator() + coop = Cooperator() work = (callable(elem, *args, **named) for elem in iterable) - return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + return DeferredList([coop.coiterate(work) for _ in range(count)]) -def process_chain(callbacks, input, *a, **kw): +def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks""" - d = defer.Deferred() + d = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) return d -def process_chain_both(callbacks, errbacks, input, *a, **kw): +def process_chain_both(callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" - d = defer.Deferred() + d = Deferred() for cb, eb in zip(callbacks, errbacks): d.addCallbacks( callback=cb, errback=eb, @@ -100,17 +112,17 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): return d -def process_parallel(callbacks, input, *a, **kw): +def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d = defer.DeferredList(dfds, fireOnOneErrback=1, consumeErrors=1) + d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d -def iter_errback(iterable, errback, *a, **kw): +def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -124,22 +136,22 @@ def iter_errback(iterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o): +def deferred_from_coro(o) -> Any: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" - if isinstance(o, defer.Deferred): + if isinstance(o, Deferred): return o if asyncio.isfuture(o) or inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" - return defer.ensureDeferred(o) + return ensureDeferred(o) else: # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - return defer.Deferred.fromFuture(asyncio.ensure_future(o)) + return Deferred.fromFuture(asyncio.ensure_future(o)) return o -def deferred_f_from_coro_f(coro_f): +def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ Converts a coroutine function into a function that returns a Deferred. The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it. @@ -151,14 +163,14 @@ def deferred_f_from_coro_f(coro_f): return f -def maybeDeferred_coro(f, *args, **kw): +def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ try: result = f(*args, **kw) except: # noqa: E722 - return defer.fail(failure.Failure(captureVars=defer.Deferred.debug)) + return defer.fail(failure.Failure(captureVars=Deferred.debug)) - if isinstance(result, defer.Deferred): + if isinstance(result, Deferred): return result elif asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) @@ -166,3 +178,55 @@ def maybeDeferred_coro(f, *args, **kw): return defer.fail(result) else: return defer.succeed(result) + + +def deferred_to_future(d: Deferred) -> Future: + """ + .. versionadded:: 2.6.0 + + Return an :class:`asyncio.Future` object that wraps *d*. + + When :ref:`using the asyncio reactor `, you cannot await + on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy + callables defined as coroutines `, you can only await on + ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects + allows you to wait on them:: + + class MySpider(Spider): + ... + async def parse(self, response): + d = treq.get('https://example.com/additional') + additional_response = await deferred_to_future(d) + """ + return d.asFuture(asyncio.get_event_loop()) + + +def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: + """ + .. versionadded:: 2.6.0 + + Return *d* as an object that can be awaited from a :ref:`Scrapy callable + defined as a coroutine `. + + What you can await in Scrapy callables defined as coroutines depends on the + value of :setting:`TWISTED_REACTOR`: + + - When not using the asyncio reactor, you can only await on + :class:`~twisted.internet.defer.Deferred` objects. + + - When :ref:`using the asyncio reactor `, you can only + await on :class:`asyncio.Future` objects. + + If you want to write code that uses ``Deferred`` objects but works with any + reactor, use this function on all ``Deferred`` objects:: + + class MySpider(Spider): + ... + async def parse(self, response): + d = treq.get('https://example.com/additional') + extra_response = await maybe_deferred_to_future(d) + """ + if not is_asyncio_reactor_installed(): + return d + else: + return deferred_to_future(d) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f5b17416f..ae727464c 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -79,7 +79,7 @@ def create_deprecated_class( # for implementation details def __instancecheck__(cls, inst): return any(cls.__subclasscheck__(c) - for c in {type(inst), inst.__class__}) + for c in (type(inst), inst.__class__)) def __subclasscheck__(cls, sub): if cls is not DeprecatedClass.deprecated_class: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 0c1cee1a0..8e3ec2c37 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -8,11 +8,10 @@ def get_engine_status(engine): """Return a report of the current engine status""" tests = [ "time()-engine.start_time", - "engine.has_capacity()", "len(engine.downloader.active)", "engine.scraper.is_idle()", "engine.spider.name", - "engine.spider_is_idle(engine.spider)", + "engine.spider_is_idle()", "engine.slot.closing", "len(engine.slot.inprogress)", "len(engine.slot.scheduler.dqs or [])", diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 11d433cf5..76156a4b8 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,7 +1,6 @@ +import struct from gzip import GzipFile from io import BytesIO -import re -import struct from scrapy.utils.decorators import deprecated @@ -42,17 +41,5 @@ def gunzip(data): return b''.join(output_list) -_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search -_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search - - -@deprecated -def is_gzipped(response): - """Return True if the response is gzipped, or False otherwise""" - ctype = response.headers.get('Content-Type', b'') - cenc = response.headers.get('Content-Encoding', b'').lower() - return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip') - - def gzip_magic_number(response): return response.body[:3] == b'\x1f\x8b\x08' diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py deleted file mode 100644 index ceb3f0509..000000000 --- a/scrapy/utils/http.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.http instead of this module -""" - -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.decorators import deprecated -from w3lib.http import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.http` is deprecated, " - "Please import from `w3lib.http` instead.", - ScrapyDeprecationWarning, stacklevel=2) - - -@deprecated -def decode_chunked_transfer(chunked_body): - """Parsed body received with chunked transfer encoding, and return the - decoded body. - - For more info see: - https://en.wikipedia.org/wiki/Chunked_transfer_encoding - - """ - body, h, t = '', '', chunked_body - while t: - h, t = t.split('\r\n', 1) - if h == '0': - break - size = int(h, 16) - body += t[:size] - t = t[size + 2:] - return body diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index c8d4391b1..a90f1d278 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,13 +1,16 @@ """Helper functions for scrapy.http objects (Request, Response)""" -import weakref -from urllib.parse import urlparse +from typing import Union +from urllib.parse import urlparse, ParseResult +from weakref import WeakKeyDictionary + +from scrapy.http import Request, Response -_urlparse_cache = weakref.WeakKeyDictionary() +_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = WeakKeyDictionary() -def urlparse_cached(request_or_response): +def urlparse_cached(request_or_response: Union[Request, Response]) -> ParseResult: """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object """ diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 4f1e601fc..c92ef36f5 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,7 +1,10 @@ import os +from typing import Optional + +from scrapy.settings import BaseSettings -def job_dir(settings): +def job_dir(settings: BaseSettings) -> Optional[str]: path = settings['JOBDIR'] if path and not os.path.exists(path): os.makedirs(path) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 62df7a6ab..78e302d19 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -46,6 +46,9 @@ DEFAULT_LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'loggers': { + 'hpack': { + 'level': 'ERROR', + }, 'scrapy': { 'level': 'DEBUG', }, @@ -121,8 +124,9 @@ def _get_handler(settings): """ Return a log handler object according to settings """ filename = settings.get('LOG_FILE') if filename: + mode = 'a' if settings.getbool('LOG_FILE_APPEND') else 'w' encoding = settings.get('LOG_ENCODING') - handler = logging.FileHandler(filename, encoding=encoding) + handler = logging.FileHandler(filename, mode=mode, encoding=encoding) elif settings.getbool('LOG_ENABLED'): handler = logging.StreamHandler() else: @@ -139,7 +143,7 @@ def _get_handler(settings): return handler -def log_scrapy_info(settings): +def log_scrapy_info(settings: Settings) -> None: logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ @@ -148,6 +152,9 @@ def log_scrapy_info(settings): if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) + + +def log_reactor_info() -> None: from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) from twisted.internet import asyncioreactor diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py deleted file mode 100644 index 9728c542a..000000000 --- a/scrapy/utils/markup.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.html instead of this module -""" -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.html import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.markup` is deprecated. " - "Please import from `w3lib.html` instead.", - ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 081cd33f1..1221b39b2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,17 +9,16 @@ from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules -from textwrap import dedent from w3lib.html import replace_entities from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.python import flatten, to_unicode -from scrapy.item import _BaseItem +from scrapy.item import Item from scrapy.utils.deprecate import ScrapyDeprecationWarning -_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes +_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes def arg_to_iter(arg): @@ -51,7 +50,7 @@ def load_object(path): return path else: raise TypeError("Unexpected argument type, expected string " - "or object, got: %s" % type(path)) + f"or object, got: {type(path)}") try: dot = path.rindex('.') @@ -139,7 +138,7 @@ def md5sum(file): def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" - return rel is not None and 'nofollow' in rel.split() + return rel is not None and 'nofollow' in rel.replace(',', ' ').split() def create_instance(objcls, settings, crawler, *args, **kwargs): @@ -227,7 +226,8 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - tree = ast.parse(dedent(inspect.getsource(callable))) + code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True @@ -242,12 +242,23 @@ def warn_on_generator_with_return_value(spider, callable): Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None """ - if is_generator_with_return_value(callable): + try: + if is_generator_with_return_value(callable): + warnings.warn( + f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' + 'a generator and includes a "return" statement with a value ' + 'different than None. This could lead to unexpected behaviour. Please see ' + 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' + 'for details about the semantics of the "return" statement within generators', + stacklevel=2, + ) + except IndentationError: + callable_name = spider.__class__.__name__ + "." + callable.__name__ warnings.warn( - f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' - 'a generator and includes a "return" statement with a value ' - 'different than None. This could lead to unexpected behaviour. Please see ' - 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' - 'for details about the semantics of the "return" statement within generators', + f'Unable to determine whether or not "{callable_name}" is a generator with a return value. ' + 'This will not prevent your code from working, but it prevents Scrapy from detecting ' + f'potential issues in your implementation of "{callable_name}". Please, report this in the ' + 'Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), ' + f'including the code of "{callable_name}"', stacklevel=2, ) diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py deleted file mode 100644 index 5dcf791b8..000000000 --- a/scrapy/utils/multipart.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Transitional module for moving to the w3lib library. - -For new code, always import from w3lib.form instead of this module -""" -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.form import * # noqa: F401 - - -warnings.warn("Module `scrapy.utils.multipart` is deprecated. " - "If you're using `encode_multipart` function, please use " - "`urllib3.filepost.encode_multipart_formdata` instead", - ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index fd13d85e3..c66af497e 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,4 @@ import os -import pickle import warnings from importlib import import_module @@ -68,18 +67,10 @@ def get_project_settings(): if settings_module_path: settings.setmodule(settings_module_path, priority='project') - pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") - if pickled_settings: - warnings.warn("Use of environment variable " - "'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' " - "is deprecated.", ScrapyDeprecationWarning) - settings.setdict(pickle.loads(pickled_settings), priority='project') - scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if k.startswith('SCRAPY_')} valid_envvars = { 'CHECK', - 'PICKLED_SETTINGS_TO_OVERRIDE', 'PROJECT', 'PYTHON_SHELL', 'SETTINGS_MODULE', diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py deleted file mode 100644 index c8c24076e..000000000 --- a/scrapy/utils/py36.py +++ /dev/null @@ -1,10 +0,0 @@ -""" -Helpers using Python 3.6+ syntax (ignore SyntaxError on import). -""" - - -async def collect_asyncgen(result): - results = [] - async for x in result: - results.append(x) - return results diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5703fd4c3..bcc12f24f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,6 +8,7 @@ import re import sys import warnings import weakref +from collections.abc import Iterable from functools import partial, wraps from itertools import chain @@ -335,15 +336,15 @@ else: gc.collect() -class MutableChain: +class MutableChain(Iterable): """ Thin wrapper around itertools.chain, allowing to add iterables "in-place" """ - def __init__(self, *args): + def __init__(self, *args: Iterable): self.data = chain.from_iterable(args) - def extend(self, *iterables): + def extend(self, *iterables: Iterable): self.data = chain(self.data, chain.from_iterable(iterables)) def __iter__(self): diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6723d9b37..96395543c 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,4 +1,5 @@ import asyncio +import sys from contextlib import suppress from twisted.internet import asyncioreactor, error @@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): + if sys.version_info >= (3, 8) and sys.platform == "win32": + policy = asyncio.get_event_loop_policy() + if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index d38b1bc4d..c254b9f82 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -1,95 +1,22 @@ -""" -Helper functions for serializing (and deserializing) requests. -""" -import inspect +import warnings +from typing import Optional -from scrapy.http import Request -from scrapy.utils.python import to_unicode -from scrapy.utils.misc import load_object +import scrapy +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.request import request_from_dict as _from_dict -def request_to_dict(request, spider=None): - """Convert Request object to a dict. - - If a spider is given, it will try to find out the name of the spider method - used in the callback and store that as the callback. - """ - cb = request.callback - if callable(cb): - cb = _find_method(spider, cb) - eb = request.errback - if callable(eb): - eb = _find_method(spider, eb) - d = { - 'url': to_unicode(request.url), # urls should be safe (safe_string_url) - 'callback': cb, - 'errback': eb, - 'method': request.method, - 'headers': dict(request.headers), - 'body': request.body, - 'cookies': request.cookies, - 'meta': request.meta, - '_encoding': request._encoding, - 'priority': request.priority, - 'dont_filter': request.dont_filter, - 'flags': request.flags, - 'cb_kwargs': request.cb_kwargs, - } - if type(request) is not Request: - d['_class'] = request.__module__ + '.' + request.__class__.__name__ - return d +warnings.warn( + ("Module scrapy.utils.reqser is deprecated, please use request.to_dict method" + " and/or scrapy.utils.request.request_from_dict instead"), + category=ScrapyDeprecationWarning, + stacklevel=2, +) -def request_from_dict(d, spider=None): - """Create Request object from a dict. - - If a spider is given, it will try to resolve the callbacks looking at the - spider for methods with the same name. - """ - cb = d['callback'] - if cb and spider: - cb = _get_method(spider, cb) - eb = d['errback'] - if eb and spider: - eb = _get_method(spider, eb) - request_cls = load_object(d['_class']) if '_class' in d else Request - return request_cls( - url=to_unicode(d['url']), - callback=cb, - errback=eb, - method=d['method'], - headers=d['headers'], - body=d['body'], - cookies=d['cookies'], - meta=d['meta'], - encoding=d['_encoding'], - priority=d['priority'], - dont_filter=d['dont_filter'], - flags=d.get('flags'), - cb_kwargs=d.get('cb_kwargs'), - ) +def request_to_dict(request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None) -> dict: + return request.to_dict(spider=spider) -def _find_method(obj, func): - # Only instance methods contain ``__func__`` - if obj and hasattr(func, '__func__'): - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name - raise ValueError(f"Function {func} is not an instance method in: {obj}") - - -def _get_method(obj, name): - name = str(name) - try: - return getattr(obj, name) - except AttributeError: - raise ValueError(f"Method {name!r} not found in: {obj}") +def request_from_dict(d: dict, spider: Optional["scrapy.Spider"] = None) -> "scrapy.Request": + return _from_dict(d, spider=spider) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 12c03d78e..cf33317ce 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -4,20 +4,156 @@ scrapy.http.Request objects """ import hashlib -import weakref +import json +import warnings +from typing import Dict, Iterable, List, Optional, Tuple, Union from urllib.parse import urlunparse +from weakref import WeakKeyDictionary from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url +from scrapy import Request, Spider +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode -_fingerprint_cache = weakref.WeakKeyDictionary() +_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" +_deprecated_fingerprint_cache = WeakKeyDictionary() -def request_fingerprint(request, include_headers=None, keep_fragments=False): +def _serialize_headers(headers, request): + for header in headers: + if header in request.headers: + yield header + for value in request.headers.getlist(header): + yield value + + +def request_fingerprint( + request: Request, + include_headers: Optional[Iterable[Union[bytes, str]]] = None, + keep_fragments: bool = False, +) -> str: + """ + Return the request fingerprint as an hexadecimal string. + + The request fingerprint is a hash that uniquely identifies the resource the + request points to. For example, take the following two urls: + + http://www.example.com/query?id=111&cat=222 + http://www.example.com/query?cat=222&id=111 + + Even though those are two different URLs both point to the same resource + and are equivalent (i.e. they should return the same response). + + Another example are cookies used to store session ids. Suppose the + following page is only accessible to authenticated users: + + http://www.example.com/members/offers.html + + Lots of sites use a cookie to store the session id, which adds a random + component to the HTTP Request and thus should be ignored when calculating + the fingerprint. + + For this reason, request headers are ignored by default when calculating + the fingerprint. If you want to include specific headers use the + include_headers argument, which is a list of Request headers to include. + + Also, servers usually ignore fragments in urls when handling requests, + so they are also ignored by default when calculating the fingerprint. + If you want to include them, set the keep_fragments argument to True + (for instance when handling requests with a headless browser). + """ + if include_headers or keep_fragments: + message = ( + 'Call to deprecated function ' + 'scrapy.utils.request.request_fingerprint().\n' + '\n' + 'If you are using this function in a Scrapy component because you ' + 'need a non-default fingerprinting algorithm, and you are OK ' + 'with that non-default fingerprinting algorithm being used by ' + 'all Scrapy components and not just the one calling this ' + 'function, use crawler.request_fingerprinter.fingerprint() ' + 'instead in your Scrapy component (you can get the crawler ' + 'object from the \'from_crawler\' class method), and use the ' + '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your ' + 'non-default fingerprinting algorithm.\n' + '\n' + 'Otherwise, consider using the ' + 'scrapy.utils.request.fingerprint() function instead.\n' + '\n' + 'If you switch to \'fingerprint()\', or assign the ' + '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses ' + '\'fingerprint()\', the generated fingerprints will not only be ' + 'bytes instead of a string, but they will also be different from ' + 'those generated by \'request_fingerprint()\'. Before you switch, ' + 'make sure that you understand the consequences of this (e.g. ' + 'cache invalidation) and are OK with them; otherwise, consider ' + 'implementing your own function which returns the same ' + 'fingerprints as the deprecated \'request_fingerprint()\' function.' + ) + else: + message = ( + 'Call to deprecated function ' + 'scrapy.utils.request.request_fingerprint().\n' + '\n' + 'If you are using this function in a Scrapy component, and you ' + 'are OK with users of your component changing the fingerprinting ' + 'algorithm through settings, use ' + 'crawler.request_fingerprinter.fingerprint() instead in your ' + 'Scrapy component (you can get the crawler object from the ' + '\'from_crawler\' class method).\n' + '\n' + 'Otherwise, consider using the ' + 'scrapy.utils.request.fingerprint() function instead.\n' + '\n' + 'Either way, the resulting fingerprints will be returned as ' + 'bytes, not as a string, and they will also be different from ' + 'those generated by \'request_fingerprint()\'. Before you switch, ' + 'make sure that you understand the consequences of this (e.g. ' + 'cache invalidation) and are OK with them; otherwise, consider ' + 'implementing your own function which returns the same ' + 'fingerprints as the deprecated \'request_fingerprint()\' function.' + ) + warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) + processed_include_headers: Optional[Tuple[bytes, ...]] = None + if include_headers: + processed_include_headers = tuple( + to_bytes(h.lower()) for h in sorted(include_headers) + ) + cache = _deprecated_fingerprint_cache.setdefault(request, {}) + cache_key = (processed_include_headers, keep_fragments) + if cache_key not in cache: + fp = hashlib.sha1() + fp.update(to_bytes(request.method)) + fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) + fp.update(request.body or b'') + if processed_include_headers: + for part in _serialize_headers(processed_include_headers, request): + fp.update(part) + cache[cache_key] = fp.hexdigest() + return cache[cache_key] + + +def _request_fingerprint_as_bytes(*args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return bytes.fromhex(request_fingerprint(*args, **kwargs)) + + +_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +_fingerprint_cache = WeakKeyDictionary() + + +def fingerprint( + request: Request, + *, + include_headers: Optional[Iterable[Union[bytes, str]]] = None, + keep_fragments: bool = False, +) -> bytes: """ Return the request fingerprint. @@ -35,47 +171,115 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): http://www.example.com/members/offers.html - Lot of sites use a cookie to store the session id, which adds a random + Lots of sites use a cookie to store the session id, which adds a random component to the HTTP Request and thus should be ignored when calculating the fingerprint. For this reason, request headers are ignored by default when calculating - the fingeprint. If you want to include specific headers use the + the fingerprint. If you want to include specific headers use the include_headers argument, which is a list of Request headers to include. Also, servers usually ignore fragments in urls when handling requests, so they are also ignored by default when calculating the fingerprint. If you want to include them, set the keep_fragments argument to True (for instance when handling requests with a headless browser). - """ + processed_include_headers: Optional[Tuple[bytes, ...]] = None if include_headers: - include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) + processed_include_headers = tuple( + to_bytes(h.lower()) for h in sorted(include_headers) + ) cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (include_headers, keep_fragments) + cache_key = (processed_include_headers, keep_fragments) if cache_key not in cache: - fp = hashlib.sha1() - fp.update(to_bytes(request.method)) - fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) - fp.update(request.body or b'') - if include_headers: - for hdr in include_headers: - if hdr in request.headers: - fp.update(hdr) - for v in request.headers.getlist(hdr): - fp.update(v) - cache[cache_key] = fp.hexdigest() + # To decode bytes reliably (JSON does not support bytes), regardless of + # character encoding, we use bytes.hex() + headers: Dict[str, List[str]] = {} + if processed_include_headers: + for header in processed_include_headers: + if header in request.headers: + headers[header.hex()] = [ + header_value.hex() + for header_value in request.headers.getlist(header) + ] + fingerprint_data = { + 'method': to_unicode(request.method), + 'url': canonicalize_url(request.url, keep_fragments=keep_fragments), + 'body': (request.body or b'').hex(), + 'headers': headers, + } + fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) + cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() return cache[cache_key] -def request_authenticate(request, username, password): - """Autenticate the given request (in place) using the HTTP basic access +class RequestFingerprinter: + """Default fingerprinter. + + It takes into account a canonical version + (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url + ` and the values of :attr:`request.method + ` and :attr:`request.body + `. It then generates an `SHA1 + `_ hash. + + .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`. + """ + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def __init__(self, crawler=None): + if crawler: + implementation = crawler.settings.get( + 'REQUEST_FINGERPRINTER_IMPLEMENTATION' + ) + else: + implementation = 'PREVIOUS_VERSION' + if implementation == 'PREVIOUS_VERSION': + message = ( + '\'PREVIOUS_VERSION\' is a deprecated value for the ' + '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n' + '\n' + 'It is also the default value. In other words, it is normal ' + 'to get this warning if you have not defined a value for the ' + '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so ' + 'for backward compatibility reasons, but it will change in a ' + 'future version of Scrapy.\n' + '\n' + 'See the documentation of the ' + '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for ' + 'information on how to handle this deprecation.' + ) + warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) + self._fingerprint = _request_fingerprint_as_bytes + elif implementation == 'VERSION': + self._fingerprint = fingerprint + else: + raise ValueError( + f'Got an invalid value on setting ' + f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': ' + f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) ' + f'and \'VERSION\'.' + ) + + def fingerprint(self, request): + return self._fingerprint(request) + + +def request_authenticate( + request: Request, + username: str, + password: str, +) -> None: + """Authenticate the given request (in place) using the HTTP basic access authentication mechanism (RFC 2617) and the given username and password """ request.headers['Authorization'] = basic_auth_header(username, password) -def request_httprepr(request): +def request_httprepr(request: Request) -> bytes: """Return the raw HTTP representation (as bytes) of the given request. This is provided only for reference since it's not the actual stream of bytes that will be send when performing the request (that's controlled @@ -92,9 +296,33 @@ def request_httprepr(request): return s -def referer_str(request): +def referer_str(request: Request) -> Optional[str]: """ Return Referer HTTP header suitable for logging. """ referrer = request.headers.get('Referer') if referrer is None: return referrer return to_unicode(referrer, errors='replace') + + +def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: + """Create a :class:`~scrapy.Request` object from a dict. + + If a spider is given, it will try to resolve the callbacks looking at the + spider for methods with the same name. + """ + request_cls = load_object(d["_class"]) if "_class" in d else Request + kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} + if d.get("callback") and spider: + kwargs["callback"] = _get_method(spider, d["callback"]) + if d.get("errback") and spider: + kwargs["errback"] = _get_method(spider, d["errback"]) + return request_cls(**kwargs) + + +def _get_method(obj, name): + """Helper function for request_from_dict""" + name = str(name) + try: + return getattr(obj, name) + except AttributeError: + raise ValueError(f"Method {name!r} not found in: {obj}") diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 99b089b6f..741dce350 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,19 +3,25 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import weakref -import webbrowser +import re import tempfile +import webbrowser +from typing import Any, Callable, Iterable, Optional, Tuple, Union +from weakref import WeakKeyDictionary + +import scrapy +from scrapy.http.response import Response from twisted.web import http from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.decorators import deprecated from w3lib import html -_baseurl_cache = weakref.WeakKeyDictionary() +_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() -def get_base_url(response): +def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] @@ -23,10 +29,13 @@ def get_base_url(response): return _baseurl_cache[response] -_metaref_cache = weakref.WeakKeyDictionary() +_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = WeakKeyDictionary() -def get_meta_refresh(response, ignore_tags=('script', 'noscript')): +def get_meta_refresh( + response: "scrapy.http.response.text.TextResponse", + ignore_tags: Optional[Iterable[str]] = ('script', 'noscript'), +) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] @@ -35,14 +44,16 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')): return _metaref_cache[response] -def response_status_message(status): +def response_status_message(status: Union[bytes, float, int, str]) -> str: """Return status code plus status text descriptive message """ - message = http.RESPONSES.get(int(status), "Unknown Status") - return f'{status} {to_unicode(message)}' + status_int = int(status) + message = http.RESPONSES.get(status_int, "Unknown Status") + return f'{status_int} {to_unicode(message)}' -def response_httprepr(response): +@deprecated +def response_httprepr(response: Response) -> bytes: """Return raw HTTP representation (as bytes) of the given response. This is provided only for reference, since it's not the exact stream of bytes that was received (that's not exposed by Twisted). @@ -60,7 +71,10 @@ def response_httprepr(response): return b"".join(values) -def open_in_browser(response, _openfunc=webbrowser.open): +def open_in_browser( + response: Union["scrapy.http.response.html.HtmlResponse", "scrapy.http.response.text.TextResponse"], + _openfunc: Callable[[str], Any] = webbrowser.open, +) -> Any: """Open the given response in a local web browser, populating the tag for external links to work """ @@ -69,8 +83,9 @@ def open_in_browser(response, _openfunc=webbrowser.open): body = response.body if isinstance(response, HtmlResponse): if b'' - body = body.replace(b'', to_bytes(repl)) + repl = fr'\1' + body = re.sub(b"", b"", body, flags=re.DOTALL) + body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 115707182..fbafc9d45 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,5 @@ """Helper functions for working with signals""" - +import collections.abc import logging from twisted.internet.defer import DeferredList, Deferred @@ -16,15 +16,13 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) -class _IgnoredException(Exception): - pass - - def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ - dont_log = (named.pop('dont_log', _IgnoredException), StopDownload) + dont_log = named.pop('dont_log', ()) + dont_log = tuple(dont_log) if isinstance(dont_log, collections.abc.Sequence) else (dont_log,) + dont_log += (StopDownload, ) spider = named.get('spider', None) responses = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index f3a9a67a3..59fc9202f 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,17 +4,14 @@ import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -try: - from scrapy.utils.py36 import collect_asyncgen -except SyntaxError: - collect_asyncgen = None +from scrapy.utils.asyncgen import collect_asyncgen logger = logging.getLogger(__name__) def iterate_spider_output(result): - if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result): + if inspect.isasyncgen(result): d = deferred_from_coro(collect_asyncgen(result)) d.addCallback(iterate_spider_output) return d diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 24c38283a..b90ea5009 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -54,7 +54,7 @@ def get_ftp_content_and_delete( return "".join(ftp_data) -def get_crawler(spidercls=None, settings_dict=None): +def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. @@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None): from scrapy.crawler import CrawlerRunner from scrapy.spiders import Spider - runner = CrawlerRunner(settings_dict) + # Set by default settings that prevent deprecation warnings. + settings = {} + if prevent_warnings: + settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION' + settings.update(settings_dict or {}) + runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 3e40acd69..b0c6a2424 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,14 +9,15 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ -import weakref -from time import time -from operator import itemgetter from collections import defaultdict +from operator import itemgetter +from time import time +from typing import DefaultDict +from weakref import WeakKeyDictionary NoneType = type(None) -live_refs = defaultdict(weakref.WeakKeyDictionary) +live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) class object_ref: diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 00226283f..f704e113f 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -260,7 +260,7 @@ ItemForm ia['width'] = x.x('//p[@class="width"]') ia['volume'] = x.x('//p[@class="volume"]') - # another example passing parametes on instance + # another example passing parameters on instance ia = NewsForm(response, encoding='utf-8') ia['name'] = x.x('//p[@class="name"]') diff --git a/sep/sep-005.rst b/sep/sep-005.rst index e795838e4..08ed367b3 100644 --- a/sep/sep-005.rst +++ b/sep/sep-005.rst @@ -107,7 +107,7 @@ gUsing default_builder This will use default_builder as the builder for every field in the item class. -As a reducer is not set reducers will be set based on Item Field classess. +As a reducer is not set reducers will be set based on Item Field classes. gReset default_builder for a field ================================== diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 8ca81824d..0859e3f7c 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -64,7 +64,7 @@ Request Processors takes requests objects and can perform any action to them, like filtering or modifying on the fly. The current ``LinkExtractor`` had integrated link processing, like -canonicalize. Request Processors can be reutilized and applied in serie. +canonicalize. Request Processors can be reutilized and applied in series. Request Generator ----------------- diff --git a/sep/sep-021.rst b/sep/sep-021.rst index 372429791..c1ec16f7f 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -22,7 +22,7 @@ Instead, the hooks are spread over: * Downloader handlers (DOWNLOADER_HANDLERS) * Item pipelines (ITEM_PIPELINES) * Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES) -* Overrideable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) +* Overridable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) * Generic extensions (EXTENSIONS) * CLI commands (COMMANDS_MODULE) diff --git a/setup.cfg b/setup.cfg index 8101443e3..1fab6fe22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,57 +10,18 @@ follow_imports = skip # FIXME: remove the following sections once the issues are solved -[mypy-scrapy] -ignore_errors = True - -[mypy-scrapy.commands] -ignore_errors = True - -[mypy-scrapy.commands.parse] -ignore_errors = True - [mypy-scrapy.downloadermiddlewares.httpproxy] ignore_errors = True -[mypy-scrapy.contracts] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True -[mypy-scrapy.item] -ignore_errors = True - -[mypy-scrapy.http.cookies] -ignore_errors = True - -[mypy-scrapy.mail] -ignore_errors = True - [mypy-scrapy.pipelines.images] ignore_errors = True [mypy-scrapy.settings.default_settings] ignore_errors = True -[mypy-scrapy.spidermiddlewares.referer] -ignore_errors = True - -[mypy-scrapy.utils.httpobj] -ignore_errors = True - -[mypy-scrapy.utils.request] -ignore_errors = True - -[mypy-scrapy.utils.response] -ignore_errors = True - -[mypy-scrapy.utils.spider] -ignore_errors = True - -[mypy-scrapy.utils.trackref] -ignore_errors = True - [mypy-tests.mocks.dummydbm] ignore_errors = True @@ -82,9 +43,6 @@ ignore_errors = True [mypy-tests.test_downloader_handlers] ignore_errors = True -[mypy-tests.test_engine] -ignore_errors = True - [mypy-tests.test_exporters] ignore_errors = True diff --git a/setup.py b/setup.py index 0c2281400..ed197273f 100644 --- a/setup.py +++ b/setup.py @@ -19,44 +19,40 @@ def has_environment_marker_platform_impl_support(): install_requires = [ - 'Twisted>=17.9.0', - 'cryptography>=2.0', + 'Twisted>=18.9.0', + 'cryptography>=2.8', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'PyDispatcher>=2.0.5', - 'pyOpenSSL>=16.2.0', + 'pyOpenSSL>=19.1.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', 'w3lib>=1.17.0', - 'zope.interface>=4.1.3', + 'zope.interface>=5.1.0', 'protego>=0.1.15', 'itemadapter>=0.1.0', + 'setuptools', + 'tldextract', + 'lxml>=4.3.0', ] extras_require = {} - +cpython_dependencies = [ + 'PyDispatcher>=2.0.5', +] if has_environment_marker_platform_impl_support(): - extras_require[':platform_python_implementation == "CPython"'] = [ - 'lxml>=3.5.0', - ] + extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies extras_require[':platform_python_implementation == "PyPy"'] = [ - # Earlier lxml versions are affected by - # https://foss.heptapod.net/pypy/pypy/-/issues/2498, - # which was fixed in Cython 0.26, released on 2017-06-19, and used to - # generate the C headers of lxml release tarballs published since then, the - # first of which was: - 'lxml>=4.0.0', 'PyPyDispatcher>=2.1.0', ] else: - install_requires.append('lxml>=3.5.0') + install_requires.extend(cpython_dependencies) setup( name='Scrapy', version=version, url='https://scrapy.org', - project_urls = { + project_urls={ 'Documentation': 'https://docs.scrapy.org/', 'Source': 'https://github.com/scrapy/scrapy', 'Tracker': 'https://github.com/scrapy/scrapy/issues', @@ -82,16 +78,17 @@ setup( 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.6', + python_requires='>=3.7', install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index 46c2a12a4..bdd3c1fef 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,5 +1,6 @@ import asyncio import sys +from typing import Optional from scrapy import Spider from scrapy.crawler import CrawlerProcess @@ -31,6 +32,7 @@ class UrlSpider(Spider): if __name__ == "__main__": + ASYNCIO_EVENT_LOOP: Optional[str] try: ASYNCIO_EVENT_LOOP = sys.argv[1] except IndexError: diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 8568bd8b8..f2a93074b 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,6 +1,9 @@ import asyncio +import sys from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) import scrapy diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..56304bd23 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py new file mode 100644 index 000000000..3f219098c --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class SelectReactorSpider(scrapy.Spider): + name = 'select_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(SelectReactorSpider) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..72bb986bc --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = 'asyncio_reactor1' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +class AsyncioReactorSpider2(scrapy.Spider): + name = 'asyncio_reactor2' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/constraints.txt b/tests/constraints.txt deleted file mode 100644 index 5655ac2d3..000000000 --- a/tests/constraints.txt +++ /dev/null @@ -1 +0,0 @@ -Twisted!=18.4.0 \ No newline at end of file diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index da202be4d..bb4a8e5af 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -40,9 +40,9 @@ def generate_keys(): subject = issuer = Name( [ - NameAttribute(NameOID.COUNTRY_NAME, u"IE"), - NameAttribute(NameOID.ORGANIZATION_NAME, u"Scrapy"), - NameAttribute(NameOID.COMMON_NAME, u"localhost"), + NameAttribute(NameOID.COUNTRY_NAME, "IE"), + NameAttribute(NameOID.ORGANIZATION_NAME, "Scrapy"), + NameAttribute(NameOID.COMMON_NAME, "localhost"), ] ) cert = ( @@ -54,7 +54,7 @@ def generate_keys(): .not_valid_before(datetime.utcnow()) .not_valid_after(datetime.utcnow() + timedelta(days=10)) .add_extension( - SubjectAlternativeName([DNSName(u"localhost")]), + SubjectAlternativeName([DNSName("localhost")]), critical=False, ) .sign(key, SHA256(), default_backend()) diff --git a/tests/mockserver.py b/tests/mockserver.py index ab9aec6a6..72d7e0241 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -14,10 +14,9 @@ from twisted.internet import defer, reactor, ssl from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory -from twisted.web.resource import EncodingResourceWrapper, Resource +from twisted.web import resource, server from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site from twisted.web.static import File -from twisted.web.test.test_webclient import PayloadResource from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode @@ -35,7 +34,70 @@ def getarg(request, name, default=None, type=None): return default -class LeafResource(Resource): +# most of the following resources are copied from twisted.web.test.test_webclient +class ForeverTakingResource(resource.Resource): + """ + L{ForeverTakingResource} is a resource which never finishes responding + to requests. + """ + + def __init__(self, write=False): + resource.Resource.__init__(self) + self._write = write + + def render(self, request): + if self._write: + request.write(b"some bytes") + return server.NOT_DONE_YET + + +class ErrorResource(resource.Resource): + def render(self, request): + request.setResponseCode(401) + if request.args.get(b"showlength"): + request.setHeader(b"content-length", b"0") + return b"" + + +class NoLengthResource(resource.Resource): + def render(self, request): + return b"nolength" + + +class HostHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the host header + from the request. + """ + + def render(self, request): + return request.requestHeaders.getRawHeaders(b"host")[0] + + +class PayloadResource(resource.Resource): + """ + A testing resource which renders itself as the contents of the request body + as long as the request body is 100 bytes long, otherwise which renders + itself as C{"ERROR"}. + """ + + def render(self, request): + data = request.content.read() + contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] + if len(data) != 100 or int(contentLength) != 100: + return b"ERROR" + return data + + +class BrokenDownloadResource(resource.Resource): + def render(self, request): + # only sends 3 bytes even though it claims to send 5 + request.setHeader(b"content-length", b"5") + request.write(b"abc") + return b"" + + +class LeafResource(resource.Resource): isLeaf = True @@ -175,10 +237,10 @@ class ArbitraryLengthPayloadResource(LeafResource): return request.content.read() -class Root(Resource): +class Root(resource.Resource): def __init__(self): - Resource.__init__(self) + resource.Resource.__init__(self) self.putChild(b"status", Status()) self.putChild(b"follow", Follow()) self.putChild(b"delay", Delay()) @@ -187,7 +249,7 @@ class Root(Resource): self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) self.putChild(b"payload", PayloadResource()) - self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"xpayload", resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) try: from tests import tests_datadir diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt deleted file mode 100644 index 7f8a5c52e..000000000 --- a/tests/requirements-py3.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Tests requirements -attrs -dataclasses; python_version == '3.6' -mitmproxy; python_version >= '3.7' -mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' -pyftpdlib -# https://github.com/pytest-dev/pytest-twisted/issues/93 -pytest != 5.4, != 5.4.1 -pytest-cov -pytest-twisted >= 1.11 -pytest-xdist -sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 -testfixtures -uvloop; platform_system != "Windows" - -# optional for shell wrapper tests -bpython -brotlipy # optional for HTTP compress downloader middleware tests -zstandard # optional for HTTP compress downloader middleware tests -ipython -pywin32; sys_platform == "win32" diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 000000000..d9373dfa8 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,16 @@ +# Tests requirements +attrs +pyftpdlib +pytest +pytest-cov==3.0.0 +pytest-xdist +sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 +testfixtures +uvloop; platform_system != "Windows" + +# optional for shell wrapper tests +bpython +brotli # optional for HTTP compress downloader middleware tests +zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests +ipython +pywin32; sys_platform == "win32" diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html index 2307ea865..e3a2a4145 100644 --- a/tests/sample_data/link_extractor/linkextractor.html +++ b/tests/sample_data/link_extractor/linkextractor.html @@ -1,20 +1,22 @@ + + - - -Sample page with links for testing LinkExtractor - - - - - + + + Sample page with links for testing LinkExtractor + + + + + \ No newline at end of file diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html index e7eee18de..1e05bf0f0 100644 --- a/tests/sample_data/link_extractor/linkextractor_latin1.html +++ b/tests/sample_data/link_extractor/linkextractor_latin1.html @@ -1,3 +1,5 @@ + + @@ -7,11 +9,11 @@ diff --git a/tests/sample_data/link_extractor/linkextractor_no_href.html b/tests/sample_data/link_extractor/linkextractor_no_href.html index 0b01cede8..2d67ec6ff 100644 --- a/tests/sample_data/link_extractor/linkextractor_no_href.html +++ b/tests/sample_data/link_extractor/linkextractor_no_href.html @@ -1,3 +1,5 @@ + + @@ -21,5 +23,4 @@ - \ No newline at end of file diff --git a/tests/sample_data/link_extractor/linkextractor_noenc.html b/tests/sample_data/link_extractor/linkextractor_noenc.html index f9166adbe..6fa137cd9 100644 --- a/tests/sample_data/link_extractor/linkextractor_noenc.html +++ b/tests/sample_data/link_extractor/linkextractor_noenc.html @@ -1,14 +1,17 @@ + + - - -Sample page without encoding for testing LinkExtractor - + + + Sample page without encoding for testing LinkExtractor + + -
-
- -
-sample € text -
+
+
+ sample2 +
+ sample € text +
diff --git a/tests/sample_data/test_site/index.html b/tests/sample_data/test_site/index.html index d268c846a..afe17d8e2 100644 --- a/tests/sample_data/test_site/index.html +++ b/tests/sample_data/test_site/index.html @@ -1,18 +1,15 @@ + + - - -Scrapy test site - - - - -

Scrapy test site

- - - - - + + Scrapy test site + + +

Scrapy test site

+ + + \ No newline at end of file diff --git a/tests/sample_data/test_site/item1.html b/tests/sample_data/test_site/item1.html index ceeb6dc87..ee39f16f3 100644 --- a/tests/sample_data/test_site/item1.html +++ b/tests/sample_data/test_site/item1.html @@ -1,17 +1,14 @@ + + - - -Item 1 - Scrapy test site - - - - -

Item 1 name

- -
    -
  • Price: $100
    • -
  • Stock: 12
    • -
- - + + Item 1 - Scrapy test site + + +

Item 1 name

+
    +
  • Price: $100
    • +
  • Stock: 12
    • +
+ diff --git a/tests/sample_data/test_site/item2.html b/tests/sample_data/test_site/item2.html index a64c92810..f40f70750 100644 --- a/tests/sample_data/test_site/item2.html +++ b/tests/sample_data/test_site/item2.html @@ -1,17 +1,14 @@ + + - - -Item 2 - Scrapy test site - - - - -

Item 2 name

- -
    -
  • Price: $200
    • -
  • Stock: 5
    • -
- - - + + Item 2 - Scrapy test site + + +

Item 2 name

+
    +
  • Price: $200
    • +
  • Stock: 5
    • +
+ + \ No newline at end of file diff --git a/tests/spiders.py b/tests/spiders.py index 106392ea6..67dbbbe0f 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -45,7 +45,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") self.start_urls = [url] def parse(self, response): @@ -86,7 +86,7 @@ class SimpleSpider(MetaSpider): self.start_urls = [url] def parse(self, response): - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefSpider(SimpleSpider): @@ -95,7 +95,7 @@ class AsyncDefSpider(SimpleSpider): async def parse(self, response): await defer.succeed(42) - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioSpider(SimpleSpider): @@ -105,7 +105,7 @@ class AsyncDefAsyncioSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") class AsyncDefAsyncioReturnSpider(SimpleSpider): @@ -115,7 +115,7 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") return [{'id': 1}, {'id': 2}] @@ -126,7 +126,7 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.1) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d" % status) + self.logger.info(f"Got response {status}") return {"foo": 42} @@ -138,7 +138,7 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): await asyncio.sleep(0.2) req_id = response.meta.get('req_id', 0) status = await get_from_asyncio_queue(response.status) - self.logger.info("Got response %d, req_id %d" % (status, req_id)) + self.logger.info(f"Got response {status}, req_id {req_id}") if req_id > 0: return reqs = [] @@ -155,7 +155,7 @@ class AsyncDefAsyncioGenSpider(SimpleSpider): async def parse(self, response): await asyncio.sleep(0.2) yield {'foo': 42} - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenLoopSpider(SimpleSpider): @@ -166,7 +166,7 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider): for i in range(10): await asyncio.sleep(0.1) yield {'foo': i} - self.logger.info("Got response %d" % response.status) + self.logger.info(f"Got response {response.status}") class AsyncDefAsyncioGenComplexSpider(SimpleSpider): @@ -245,7 +245,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 @@ -390,3 +390,32 @@ class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider): def bytes_received(self, data, request, spider): self.meta["bytes_received"] = data raise StopDownload(fail=True) + + +class HeadersReceivedCallbackSpider(MetaSpider): + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + crawler.signals.connect(spider.headers_received, signals.headers_received) + return spider + + def start_requests(self): + yield Request(self.mockserver.url("/status"), errback=self.errback) + + def parse(self, response): + self.meta["response"] = response + + def errback(self, failure): + self.meta["failure"] = failure + + def headers_received(self, headers, body_length, request, spider): + self.meta["headers_received"] = headers + raise StopDownload(fail=False) + + +class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider): + + def headers_received(self, headers, body_length, request, spider): + self.meta["headers_received"] = headers + raise StopDownload(fail=True) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 5ec5e2989..be8adadb3 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -41,7 +41,7 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') - key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__) + key = f'spider_exceptions/{crawler.spider.exception_cls.__name__}' errorcount = crawler.stats.get_value(key) self.assertTrue(errorcount >= close_on) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 591075a98..8233e0101 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -64,3 +64,7 @@ class CmdlineTest(unittest.TestCase): settingsdict = json.loads(settingsstr) self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) self.assertEqual(200, settingsdict[EXT_PATH]) + + def test_pathlib_path_as_feeds_key(self): + self.assertEqual(self._execute('settings', '--get', 'FEEDS'), + json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}})) diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py index 8a719ddf2..b0ac6e98b 100644 --- a/tests/test_cmdline/settings.py +++ b/tests/test_cmdline/settings.py @@ -1,5 +1,14 @@ +from pathlib import Path + EXTENSIONS = { 'tests.test_cmdline.extensions.TestExtension': 0, } TEST1 = 'default' + +FEEDS = { + Path('items.csv'): { + 'format': 'csv', + 'fields': ['price', 'name'], + }, +} diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 34f5e59dd..c3d705194 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -19,11 +19,11 @@ import scrapy class CheckSpider(scrapy.Spider): name = '{self.spider_name}' - start_urls = ['http://example.com'] + start_urls = ['http://toscrape.com'] def parse(self, response, **cb_kwargs): \"\"\" - @url http://example.com + @url http://toscrape.com {contracts} \"\"\" {parse_def} diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index ed3848d88..0d992be56 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,6 +1,10 @@ import os +import argparse from os.path import join, abspath, isfile, exists + from twisted.internet import defer +from scrapy.commands import parse +from scrapy.settings import Settings from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest from scrapy.utils.python import to_unicode @@ -219,6 +223,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) + @defer.inlineCallbacks + def test_crawlspider_not_exists_with_not_matched_url(self): + status, out, stderr = yield self.execute([self.url('/invalid_url')]) + self.assertEqual(status, 0) + @defer.inlineCallbacks def test_output_flag(self): """Checks if a file was created successfully having @@ -239,3 +248,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} content = '[\n{},\n{"foo": "bar"}\n]' with open(file_path, 'r') as f: self.assertEqual(f.read(), content) + + def test_parse_add_options(self): + command = parse.Command() + command.settings = Settings() + parser = argparse.ArgumentParser( + prog='scrapy', formatter_class=argparse.HelpFormatter, + conflict_handler='resolve', prefix_chars='-' + ) + command.add_options(parser) + namespace = parser.parse_args( + ['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name] + ) + self.assertTrue(namespace.nolinks) + self.assertEqual(namespace.depth, 2) + self.assertEqual(namespace.spider, self.spider_name) + self.assertTrue(namespace.verbose) diff --git a/tests/test_commands.py b/tests/test_commands.py index d3ac05eac..7cd19b29a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,8 +1,9 @@ import inspect import json -import optparse +import argparse import os import platform +import re import subprocess import sys import tempfile @@ -17,10 +18,12 @@ from threading import Timer from unittest import skipIf from pytest import mark +from twisted import version as twisted_version +from twisted.python.versions import Version from twisted.trial import unittest import scrapy -from scrapy.commands import ScrapyCommand +from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -34,19 +37,28 @@ class CommandSettings(unittest.TestCase): def setUp(self): self.command = ScrapyCommand() self.command.settings = Settings() - self.parser = optparse.OptionParser( - formatter=optparse.TitledHelpFormatter(), - conflict_handler='resolve', - ) + self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, + conflict_handler='resolve') self.command.add_options(self.parser) def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' - opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) + opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) + def test_help_formatter(self): + formatter = ScrapyHelpFormatter(prog='scrapy') + part_strings = ['usage: scrapy genspider [options] \n\n', + '\n', 'optional arguments:\n', '\n', 'Global Options:\n'] + self.assertEqual( + formatter._join_parts(part_strings), + ('Usage\n=====\n scrapy genspider [options] \n\n\n' + 'Optional Arguments\n==================\n\n' + 'Global Options\n--------------\n') + ) + class ProjectTest(unittest.TestCase): project_name = 'testproject' @@ -69,9 +81,14 @@ class ProjectTest(unittest.TestCase): def proc(self, *new_args, **popen_kwargs): args = (sys.executable, '-m', 'scrapy.cmdline') + new_args - p = subprocess.Popen(args, cwd=self.cwd, env=self.env, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **popen_kwargs) + p = subprocess.Popen( + args, + cwd=popen_kwargs.pop('cwd', self.cwd), + env=self.env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **popen_kwargs, + ) def kill_proc(): p.kill() @@ -87,11 +104,23 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) + def find_in_file(self, filename, regex): + """Find first pattern occurrence in file""" + pattern = re.compile(regex) + with open(filename, "r") as f: + for line in f: + match = pattern.search(line) + if match is not None: + return match + class StartprojectTest(ProjectTest): def test_startproject(self): - self.assertEqual(0, self.call('startproject', self.project_name)) + p, out, err = self.proc('startproject', self.project_name) + print(out) + print(err, file=sys.stderr) + self.assertEqual(p.returncode, 0) assert exists(join(self.proj_path, 'scrapy.cfg')) assert exists(join(self.proj_path, 'testproject')) @@ -126,6 +155,25 @@ class StartprojectTest(ProjectTest): self.assertEqual(2, self.call('startproject')) self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) + def test_existing_project_dir(self): + project_dir = mkdtemp() + project_name = self.project_name + '_existing' + project_path = os.path.join(project_dir, project_name) + os.mkdir(project_path) + + p, out, err = self.proc('startproject', project_name, cwd=project_dir) + print(out) + print(err, file=sys.stderr) + self.assertEqual(p.returncode, 0) + + assert exists(join(abspath(project_path), 'scrapy.cfg')) + assert exists(join(abspath(project_path), project_name)) + assert exists(join(join(abspath(project_path), project_name), '__init__.py')) + assert exists(join(join(abspath(project_path), project_name), 'items.py')) + assert exists(join(join(abspath(project_path), project_name), 'pipelines.py')) + assert exists(join(join(abspath(project_path), project_name), 'settings.py')) + assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py')) + def get_permissions_dict(path, renamings=None, ignore=None): @@ -453,6 +501,26 @@ class GenspiderCommandTest(CommandTest): def test_same_filename_as_existing_spider_force(self): self.test_same_filename_as_existing_spider(force=True) + def test_url(self, url='test.com', domain="test.com"): + self.assertEqual(0, self.call('genspider', '--force', 'test_name', url)) + self.assertEqual(domain, + self.find_in_file(join(self.proj_mod_path, + 'spiders', 'test_name.py'), + r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) + self.assertEqual(f'http://{domain}/', + self.find_in_file(join(self.proj_mod_path, + 'spiders', 'test_name.py'), + r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) + + def test_url_schema(self): + self.test_url('http://test.com', 'test.com') + + def test_url_path(self): + self.test_url('test.com/some/other/page', 'test.com') + + def test_url_schema_path(self): + self.test_url('https://test.com/some/other/page', 'test.com') + class GenspiderStandaloneCommandTest(ProjectTest): @@ -615,9 +683,6 @@ class MySpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' @@ -630,6 +695,7 @@ class MySpider(scrapy.Spider): @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_asyncio_loop_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', @@ -639,16 +705,16 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_custom_asyncio_loop_enabled_false(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' ]) import asyncio - loop = asyncio.new_event_loop() - self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log) + if sys.platform != 'win32': + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() + self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) def test_output(self): spider_code = """ @@ -705,6 +771,7 @@ class MySpider(scrapy.Spider): self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) +@skipIf(platform.system() != 'Windows', "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = 'myspider.pyw' @@ -717,35 +784,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_run_good_spider(self): super().test_run_good_spider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider(self): super().test_runspider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_dnscache_disabled(self): super().test_runspider_dnscache_disabled() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_level(self): super().test_runspider_log_level() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_short_names(self): super().test_runspider_log_short_names() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_no_spider_found(self): super().test_runspider_no_spider_found() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_output(self): super().test_output() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_overwrite_output(self): super().test_overwrite_output() @@ -762,6 +821,21 @@ class BenchCommandTest(CommandTest): self.assertNotIn('Unhandled Error', log) +class ViewCommandTest(CommandTest): + + def test_methods(self): + command = view.Command() + command.settings = Settings() + parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-', + formatter_class=ScrapyHelpFormatter, + conflict_handler='resolve') + command.add_options(parser) + self.assertEqual(command.short_desc(), + "Open URL in browser, as seen by Scrapy") + self.assertIn("URL using the Scrapy downloader and show its", + command.long_desc()) + + class CrawlCommandTest(CommandTest): def crawl(self, code, args=()): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1083c1678..7bda3bef2 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -35,6 +35,8 @@ from tests.spiders import ( DelaySpider, DuplicateStartRequestsSpider, FollowAllSpider, + HeadersReceivedCallbackSpider, + HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, ) @@ -272,6 +274,28 @@ with multiples lines self.assertEqual(s['engine.spider.name'], crawler.spider.name) self.assertEqual(s['len(engine.scraper.slot.active)'], 1) + @defer.inlineCallbacks + def test_format_engine_status(self): + from scrapy.utils.engine import format_engine_status + est = [] + + def cb(response): + est.append(format_engine_status(crawler.engine)) + + crawler = self.runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) + self.assertEqual(len(est), 1, est) + est = est[0].split("\n")[2:-2] # remove header & footer + # convert to dict + est = [x.split(":") for x in est] + est = [x for sublist in est for x in sublist] # flatten + est = [x.lstrip().rstrip() for x in est] + it = iter(est) + s = dict(zip(it, it)) + + self.assertEqual(s['engine.spider.name'], crawler.spider.name) + self.assertEqual(s['len(engine.scraper.slot.active)'], '1') + @defer.inlineCallbacks def test_graceful_crawl_error_handling(self): """ @@ -496,7 +520,7 @@ class CrawlSpiderTestCase(TestCase): self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) @defer.inlineCallbacks - def test_stop_download_callback(self): + def test_bytes_received_stop_download_callback(self): crawler = self.runner.create_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) @@ -505,7 +529,7 @@ class CrawlSpiderTestCase(TestCase): self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length) @defer.inlineCallbacks - def test_stop_download_errback(self): + def test_bytes_received_stop_download_errback(self): crawler = self.runner.create_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("response")) @@ -518,3 +542,23 @@ class CrawlSpiderTestCase(TestCase): self.assertLess( len(crawler.spider.meta["failure"].value.response.body), crawler.spider.full_response_length) + + @defer.inlineCallbacks + def test_headers_received_stop_download_callback(self): + crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta.get("failure")) + self.assertIsInstance(crawler.spider.meta["response"], Response) + self.assertEqual(crawler.spider.meta["response"].headers, crawler.spider.meta.get("headers_received")) + + @defer.inlineCallbacks + def test_headers_received_stop_download_errback(self): + crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta.get("response")) + self.assertIsInstance(crawler.spider.meta["failure"], Failure) + self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) + self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) + self.assertEqual( + crawler.spider.meta["failure"].value.response.headers, + crawler.spider.meta.get("headers_received")) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ab113710d..f7aa769e4 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,11 +4,11 @@ import platform import subprocess import sys import warnings -from unittest import skipIf from pytest import raises, mark -from testfixtures import LogCapture +from twisted import version as twisted_version from twisted.internet import defer +from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -96,13 +96,16 @@ class CrawlerLoggingTestCase(unittest.TestCase): def test_spider_custom_settings_log_level(self): log_file = self.mktemp() + with open(log_file, 'wb') as fo: + fo.write('previous message\n'.encode('utf-8')) class MySpider(scrapy.Spider): name = 'spider' custom_settings = { 'LOG_LEVEL': 'INFO', 'LOG_FILE': log_file, - # disable telnet if not available to avoid an extra warning + # settings to avoid extra warnings + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, } @@ -117,8 +120,9 @@ class CrawlerLoggingTestCase(unittest.TestCase): logging.error('error message') with open(log_file, 'rb') as fo: - logged = fo.read().decode('utf8') + logged = fo.read().decode('utf-8') + self.assertIn('previous message', logged) self.assertNotIn('debug message', logged) self.assertIn('info message', logged) self.assertIn('warning message', logged) @@ -129,6 +133,30 @@ class CrawlerLoggingTestCase(unittest.TestCase): crawler.stats.get_value('log_count/INFO') - info_count, 1) self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) + def test_spider_custom_settings_log_append(self): + log_file = self.mktemp() + with open(log_file, 'wb') as fo: + fo.write('previous message\n'.encode('utf-8')) + + class MySpider(scrapy.Spider): + name = 'spider' + custom_settings = { + 'LOG_FILE': log_file, + 'LOG_FILE_APPEND': False, + # disable telnet if not available to avoid an extra warning + 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, + } + + configure_logging() + Crawler(MySpider, {}) + logging.debug('debug message') + + with open(log_file, 'rb') as fo: + logged = fo.read().decode('utf-8') + + self.assertNotIn('previous message', logged) + self.assertIn('debug message', logged) + class SpiderLoaderWithWrongInterface: @@ -242,6 +270,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertEqual(runner.bootstrap_failed, True) + @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ @@ -250,35 +279,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - CrawlerRunner(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - - @defer.inlineCallbacks - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") - def test_crawler_process_asyncio_enabled_true(self): - with LogCapture(level=logging.DEBUG) as log: - if self.reactor_pytest == 'asyncio': - runner = CrawlerProcess(settings={ + runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) yield runner.crawl(NoRequestsSpider) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) - else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - - @defer.inlineCallbacks - def test_crawler_process_asyncio_enabled_false(self): - runner = CrawlerProcess(settings={"TWISTED_REACTOR": None}) - with LogCapture(level=logging.DEBUG) as log: - yield runner.crawl(NoRequestsSpider) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) class ScriptRunnerMixin: @@ -299,17 +303,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -348,16 +346,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + def test_reactor_asyncio_custom_settings(self): + log = self.run_script("twisted_reactor_custom_settings.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_reactor_asyncio_custom_settings_same(self): + log = self.run_script("twisted_reactor_custom_settings_same.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -366,6 +377,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly") @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows") + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) @@ -373,9 +385,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 5d0a1d0c9..5e63ebffb 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,8 +1,14 @@ +import os +import re +from configparser import ConfigParser from importlib import import_module + +from twisted import version as twisted_version from twisted.trial import unittest class ScrapyUtilsTest(unittest.TestCase): + def test_required_openssl_version(self): try: module = import_module('OpenSSL') @@ -13,6 +19,32 @@ class ScrapyUtilsTest(unittest.TestCase): installed_version = [int(x) for x in module.__version__.split('.')[:2]] assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" + def test_pinned_twisted_version(self): + """When running tests within a Tox environment with pinned + dependencies, make sure that the version of Twisted is the pinned + version. + + See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 + """ + if not os.environ.get('_SCRAPY_PINNED', None): + self.skipTest('Not in a pinned environment') + + tox_config_file_path = os.path.join( + os.path.dirname(__file__), + '..', + 'tox.ini', + ) + config_parser = ConfigParser() + config_parser.read(tox_config_file_path) + pattern = r'Twisted\[http2\]==([\d.]+)' + match = re.search(pattern, config_parser['pinned']['deps']) + pinned_twisted_version_string = match[1] + + self.assertEqual( + twisted_version.short(), + pinned_twisted_version_string + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 3e8d7e6b9..2bb53950d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,7 +1,9 @@ import contextlib import os import shutil +import sys import tempfile +from typing import Optional, Type from unittest import mock from testfixtures import LogCapture @@ -13,8 +15,6 @@ from twisted.trial import unittest from twisted.web import resource, server, static, util from twisted.web._newclient import ResponseFailed from twisted.web.http import _DataLoss -from twisted.web.test.test_webclient import (ForeverTakingResource, HostHeaderResource, - NoLengthResource, PayloadResource) from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers @@ -24,7 +24,6 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler - from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Headers, Request from scrapy.http.response.text import TextResponse @@ -33,8 +32,15 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto - -from tests.mockserver import MockServer, ssl_context_factory, Echo +from tests.mockserver import ( + Echo, + ForeverTakingResource, + HostHeaderResource, + MockServer, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) from tests.spiders import SingleRequestSpider @@ -115,6 +121,7 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.status, 200) self.assertEqual(response.body, b'0123456789') + self.assertEqual(response.protocol, None) request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -131,6 +138,7 @@ class ContentLengthHeaderResource(resource.Resource): A testing resource which renders itself as the value of the Content-Length header from the request. """ + def render(self, request): return request.requestHeaders.getRawHeaders(b"content-length")[0] @@ -142,6 +150,7 @@ class ChunkedResource(resource.Resource): request.write(b"chunked ") request.write(b"content\n") request.finish() + reactor.callLater(0, response) return server.NOT_DONE_YET @@ -155,6 +164,7 @@ class BrokenChunkedResource(resource.Resource): # Disable terminating chunk on finish. request.chunked = False closeConnection(request) + reactor.callLater(0, response) return server.NOT_DONE_YET @@ -186,6 +196,7 @@ class EmptyContentTypeHeaderResource(resource.Resource): A testing resource which renders itself as the value of request body without content-type header in response. """ + def render(self, request): request.setHeader("content-type", "") return request.content.read() @@ -197,14 +208,14 @@ class LargeChunkedFileResource(resource.Resource): for i in range(1024): request.write(b"x" * 1024) request.finish() + reactor.callLater(0, response) return server.NOT_DONE_YET class HttpTestCase(unittest.TestCase): - scheme = 'http' - download_handler_cls = HTTPDownloadHandler + download_handler_cls: Type = HTTPDownloadHandler # only used for HTTPS tests keyfile = 'keys/localhost.key' @@ -232,8 +243,10 @@ class HttpTestCase(unittest.TestCase): self.wrapper = WrappingFactory(self.site) self.host = 'localhost' if self.scheme == 'https': + # Using WrappingFactory do not enable HTTP/2 failing all the + # tests with H2DownloadHandler self.port = reactor.listenSSL( - 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile), + 0, self.site, ssl_context_factory(self.keyfile, self.certfile), interface=self.host) else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) @@ -281,18 +294,29 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects but no data is received spider = Spider('foo') - meta = {'download_timeout': 0.2} + meta = {'download_timeout': 0.5} request = Request(self.getURL('wait'), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) # client connects, server send headers and some body bytes but hangs spider = Spider('foo') - meta = {'download_timeout': 0.2} + meta = {'download_timeout': 0.5} request = Request(self.getURL('hang-after-headers'), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) @@ -307,16 +331,18 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): - def _test(response): - self.assertEqual(response.body, b'example.com') - self.assertEqual(request.headers.get('Host'), b'example.com') + host = self.host + ':' + str(self.portno) - request = Request(self.getURL('host'), headers={'Host': 'example.com'}) + def _test(response): + self.assertEqual(response.body, host.encode()) + self.assertEqual(request.headers.get('Host'), host.encode()) + + request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEqual, b'example.com') + d.addCallback(self.assertEqual, b'localhost') return d def test_content_length_zero_bodyless_post_request_headers(self): @@ -330,10 +356,11 @@ class HttpTestCase(unittest.TestCase): https://github.com/kennethreitz/requests/issues/405 https://bugs.python.org/issue14721 """ + def _test(response): self.assertEqual(response.body, b'0') - request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) + request = Request(self.getURL('contentlength'), method='POST') return self.download_request(request, Spider('foo')).addCallback(_test) def test_content_length_zero_bodyless_post_only_one(self): @@ -355,10 +382,24 @@ class HttpTestCase(unittest.TestCase): d.addCallback(self.assertEqual, body) return d + def test_response_header_content_length(self): + request = Request(self.getURL("file"), method=b"GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.headers[b'content-length']) + d.addCallback(self.assertEqual, b'159') + return d + class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" - download_handler_cls = HTTP10DownloadHandler + download_handler_cls: Type = HTTP10DownloadHandler + + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.0") + return d class Https10TestCase(Http10TestCase): @@ -367,7 +408,7 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler def test_download_without_maxsize_limit(self): request = Request(self.getURL('file')) @@ -489,6 +530,13 @@ class Http11TestCase(HttpTestCase): def test_download_broken_chunked_content_allow_data_loss_via_setting(self): return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked') + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "HTTP/1.1") + return d + class Https11TestCase(Http11TestCase): scheme = 'https' @@ -554,7 +602,7 @@ class Https11InvalidDNSPattern(Https11TestCase): class Https11CustomCiphers(unittest.TestCase): scheme = 'https' - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler keyfile = 'keys/localhost.key' certfile = 'keys/localhost.crt' @@ -565,10 +613,9 @@ class Https11CustomCiphers(unittest.TestCase): FilePath(self.tmpname).child("file").setContent(b"0123456789") r = static.File(self.tmpname) self.site = server.Site(r, timeout=None) - self.wrapper = WrappingFactory(self.site) self.host = 'localhost' self.port = reactor.listenSSL( - 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), + 0, self.site, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), interface=self.host) self.portno = self.port.getHost().port crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}) @@ -595,6 +642,7 @@ class Https11CustomCiphers(unittest.TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" + settings_dict: Optional[dict] = None def setUp(self): self.mockserver = MockServer() @@ -605,7 +653,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_with_content_length(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000})) @@ -614,7 +662,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) failure = crawler.spider.meta.get('failure') self.assertTrue(failure is None) @@ -623,7 +671,7 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_gzip_response(self): - crawler = get_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) body = b'1' * 100 # PayloadResource requires body length to be 100 request = Request(self.mockserver.url('/payload'), method='POST', body=body, meta={'download_maxsize': 50}) @@ -661,7 +709,8 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): - download_handler_cls = HTTPDownloadHandler + download_handler_cls: Type = HTTPDownloadHandler + expected_http_proxy_request_body = b'http://example.com' def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -684,7 +733,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'http://example.com') + self.assertEqual(response.body, self.expected_http_proxy_request_body) http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -713,14 +762,14 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): - download_handler_cls = HTTP10DownloadHandler + download_handler_cls: Type = HTTP10DownloadHandler def test_download_with_proxy_https_noconnect(self): raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') class Http11ProxyTestCase(HttpProxyTestCase): - download_handler_cls = HTTP11DownloadHandler + download_handler_cls: Type = HTTP11DownloadHandler @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -733,6 +782,16 @@ class Http11ProxyTestCase(HttpProxyTestCase): timeout = yield self.assertFailure(d, error.TimeoutError) self.assertIn(domain, timeout.osError) + def test_download_with_proxy_without_http_scheme(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, self.expected_http_proxy_request_body) + + http_proxy = self.getURL('').replace('http://', '') + request = Request('http://example.com', meta={'proxy': http_proxy}) + return self.download_request(request, Spider('foo')).addCallback(_test) + class HttpDownloadHandlerMock: @@ -768,7 +827,7 @@ class S3AnonTestCase(unittest.TestCase): class S3TestCase(unittest.TestCase): - download_handler_cls = S3DownloadHandler + download_handler_cls: Type = S3DownloadHandler # test use same example keys than amazon developer guide # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf @@ -908,7 +967,6 @@ class S3TestCase(unittest.TestCase): class BaseFTPTestCase(unittest.TestCase): - username = "scrapy" password = "passwd" req_meta = {"ftp_user": username, "ftp_password": password} @@ -946,6 +1004,7 @@ class BaseFTPTestCase(unittest.TestCase): def _clean(data): self.download_handler.client.transport.loseConnection() return data + deferred.addCallback(_clean) if callback: deferred.addCallback(callback) @@ -962,6 +1021,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) + self.assertIsNone(r.protocol) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -975,6 +1035,7 @@ class BaseFTPTestCase(unittest.TestCase): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'Moooooooooo power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']}) + return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): @@ -984,6 +1045,7 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.status, 404) + return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): @@ -1004,12 +1066,17 @@ class BaseFTPTestCase(unittest.TestCase): with open(local_fname, "rb") as f: self.assertEqual(f.read(), b"I have the power!") os.remove(local_fname) + return self._add_test_callbacks(d, _test) class FTPTestCase(BaseFTPTestCase): def test_invalid_credentials(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) from twisted.protocols.ftp import ConnectionLost meta = dict(self.req_meta) @@ -1020,11 +1087,11 @@ class FTPTestCase(BaseFTPTestCase): def _test(r): self.assertEqual(r.type, ConnectionLost) + return self._add_test_callbacks(d, errback=_test) class AnonymousFTPTestCase(BaseFTPTestCase): - username = "anonymous" req_meta = {} @@ -1120,3 +1187,10 @@ class DataURITestCase(unittest.TestCase): request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') return self.download_request(request, self.spider).addCallback(_test) + + def test_protocol(self): + def _test(response): + self.assertIsNone(response.protocol) + + request = Request("data:,") + return self.download_request(request, self.spider).addCallback(_test) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py new file mode 100644 index 000000000..3a9db3ee5 --- /dev/null +++ b/tests/test_downloader_handlers_http2.py @@ -0,0 +1,263 @@ +import json +from unittest import mock, skipIf + +from pytest import mark +from testfixtures import LogCapture +from twisted.internet import defer, error, reactor +from twisted.trial import unittest +from twisted.web import server +from twisted.web.error import SchemeNotSupported +from twisted.web.http import H2_ENABLED + +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.misc import create_instance +from scrapy.utils.test import get_crawler +from tests.mockserver import ssl_context_factory +from tests.test_downloader_handlers import ( + Https11TestCase, Https11CustomCiphers, + Http11MockServerTestCase, Http11ProxyTestCase, + UriResource +) + + +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") +class Https2TestCase(Https11TestCase): + + scheme = 'https' + HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" + + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler + + def test_protocol(self): + request = Request(self.getURL("host"), method="GET") + d = self.download_request(request, Spider("foo")) + d.addCallback(lambda r: r.protocol) + d.addCallback(self.assertEqual, "h2") + return d + + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch('scrapy.core.http2.stream.logger') as logger: + request = Request(self.getURL('largechunkedfile')) + + def check(logger): + logger.error.assert_called_once_with(mock.ANY) + + d = self.download_request(request, Spider('foo', download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(.1, d.callback, logger) + yield d + + @defer.inlineCallbacks + def test_unsupported_scheme(self): + request = Request("ftp://unsupported.scheme") + d = self.download_request(request, Spider("foo")) + yield self.assertFailure(d, SchemeNotSupported) + + def test_download_broken_content_cause_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_cause_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON) + + def test_concurrent_requests_same_domain(self): + spider = Spider('foo') + + request1 = Request(self.getURL('file')) + d1 = self.download_request(request1, spider) + d1.addCallback(lambda r: r.body) + d1.addCallback(self.assertEqual, b"0123456789") + + request2 = Request(self.getURL('echo'), method='POST') + d2 = self.download_request(request2, spider) + d2.addCallback(lambda r: r.headers['Content-Length']) + d2.addCallback(self.assertEqual, b"79") + + return defer.DeferredList([d1, d2]) + + @mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") + def test_connect_request(self): + request = Request(self.getURL('file'), method='CONNECT') + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b'') + return d + + def test_custom_content_length_good(self): + request = Request(self.getURL('contentlength')) + custom_content_length = str(len(request.body)) + request.headers['Content-Length'] = custom_content_length + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.text) + d.addCallback(self.assertEqual, custom_content_length) + return d + + def test_custom_content_length_bad(self): + request = Request(self.getURL('contentlength')) + actual_content_length = str(len(request.body)) + bad_content_length = str(len(request.body) + 1) + request.headers['Content-Length'] = bad_content_length + log = LogCapture() + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.text) + d.addCallback(self.assertEqual, actual_content_length) + d.addCallback( + lambda _: log.check_present( + ( + 'scrapy.core.http2.stream', + 'WARNING', + f'Ignoring bad Content-Length header ' + f'{bad_content_length!r} of request {request}, sending ' + f'{actual_content_length!r} instead', + ) + ) + ) + d.addCallback( + lambda _: log.uninstall() + ) + return d + + def test_duplicate_header(self): + request = Request(self.getURL('echo')) + header, value1, value2 = 'Custom-Header', 'foo', 'bar' + request.headers.appendlist(header, value1) + request.headers.appendlist(header, value2) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: json.loads(r.text)['headers'][header]) + d.addCallback(self.assertEqual, [value1, value2]) + return d + + +class Https2WrongHostnameTestCase(Https2TestCase): + tls_log_message = ( + 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' + 'subject "/C=XW/ST=XW/L=The ' + 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' + ) + + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = 'keys/example-com.key.pem' + certfile = 'keys/example-com.cert.pem' + + +class Https2InvalidDNSId(Https2TestCase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super(Https2InvalidDNSId, self).setUp() + self.host = '127.0.0.1' + + +class Https2InvalidDNSPattern(Https2TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + try: + from service_identity.exceptions import CertificateError # noqa: F401 + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") + self.tls_log_message = ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' + 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' + ) + super(Https2InvalidDNSPattern, self).setUp() + + +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") +class Https2CustomCiphers(Https11CustomCiphers): + scheme = 'https' + + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler + + +class Http2MockServerTestCase(Http11MockServerTestCase): + """HTTP 2.0 test case with MockServer""" + settings_dict = { + 'DOWNLOAD_HANDLERS': { + 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler' + } + } + + +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") +class Https2ProxyTestCase(Http11ProxyTestCase): + # only used for HTTPS tests + keyfile = 'keys/localhost.key' + certfile = 'keys/localhost.crt' + + scheme = 'https' + host = '127.0.0.1' + + expected_http_proxy_request_body = b'/' + + @classmethod + def setUpClass(cls): + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + cls.download_handler_cls = H2DownloadHandler + + def setUp(self): + site = server.Site(UriResource(), timeout=None) + self.port = reactor.listenSSL( + 0, site, + ssl_context_factory(self.keyfile, self.certfile), + interface=self.host + ) + self.portno = self.port.getHost().port + self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) + self.download_request = self.download_handler.download_request + + def getURL(self, path): + return f"{self.scheme}://{self.host}:{self.portno}/{path}" + + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/') + + http_proxy = f"{self.getURL('')}?noconnect" + request = Request('https://example.com', meta={'proxy': http_proxy}) + with self.assertWarnsRegex( + Warning, + r'Using HTTPS proxies in the noconnect mode is not supported by the ' + r'downloader handler.' + ): + return self.download_request(request, Spider('foo')).addCallback(_test) + + @defer.inlineCallbacks + def test_download_with_proxy_https_timeout(self): + with self.assertRaises(NotImplementedError): + yield super(Https2ProxyTestCase, self).test_download_with_proxy_https_timeout() diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 79f24c8a1..38be915f2 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -211,6 +211,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) +@mark.usefixtures('reactor_pytest') class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index aff8542e9..ba7453255 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -6,13 +6,57 @@ import pytest from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler +def _cookie_to_set_cookie_value(cookie): + """Given a cookie defined as a dictionary with name and value keys, and + optional path and domain keys, return the equivalent string that can be + associated to a ``Set-Cookie`` header.""" + decoded = {} + for key in ("name", "value", "path", "domain"): + if cookie.get(key) is None: + if key in ("name", "value"): + return + continue + if isinstance(cookie[key], (bool, float, int, str)): + decoded[key] = str(cookie[key]) + else: + try: + decoded[key] = cookie[key].decode("utf8") + except UnicodeDecodeError: + decoded[key] = cookie[key].decode("latin1", errors="replace") + + cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" + for key, value in decoded.items(): # path, domain + cookie_str += f"; {key.capitalize()}={value}" + return cookie_str + + +def _cookies_to_set_cookie_list(cookies): + """Given a group of cookie defined either as a dictionary or as a list of + dictionaries (i.e. in a format supported by the cookies parameter of + Request), return the equivalen list of strings that can be associated to a + ``Set-Cookie`` header.""" + if not cookies: + return [] + if isinstance(cookies, dict): + cookies = ({"name": k, "value": v} for k, v in cookies.items()) + return filter( + None, + ( + _cookie_to_set_cookie_value(cookie) + for cookie in cookies + ) + ) + + class CookiesMiddlewareTest(TestCase): def assertCookieValEqual(self, first, second, msg=None): @@ -23,9 +67,11 @@ class CookiesMiddlewareTest(TestCase): def setUp(self): self.spider = Spider('foo') self.mw = CookiesMiddleware() + self.redirect_middleware = RedirectMiddleware(settings=Settings()) def tearDown(self): del self.mw + del self.redirect_middleware def test_basic(self): req = Request('http://scrapytest.org/') @@ -347,3 +393,303 @@ class CookiesMiddlewareTest(TestCase): self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') self.assertCookieValEqual(req3.headers['Cookie'], 'key=') + + def test_primitive_type_cookies(self): + # Boolean + req1 = Request('http://example.org', cookies={'a': True}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'a=True') + + # Float + req2 = Request('http://example.org', cookies={'a': 9.5}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'a=9.5') + + # Integer + req3 = Request('http://example.org', cookies={'a': 10}) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], b'a=10') + + # String + req4 = Request('http://example.org', cookies={'a': 'b'}) + assert self.mw.process_request(req4, self.spider) is None + self.assertCookieValEqual(req4.headers['Cookie'], b'a=b') + + def _test_cookie_redirect( + self, + source, + target, + *, + cookies1, + cookies2, + ): + input_cookies = {'a': 'b'} + + if not isinstance(source, dict): + source = {'url': source} + if not isinstance(target, dict): + target = {'url': target} + target.setdefault('status', 301) + + request1 = Request(cookies=input_cookies, **source) + self.mw.process_request(request1, self.spider) + cookies = request1.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies1 else None) + + response = Response( + headers={ + 'Location': target['url'], + }, + **target, + ) + self.assertEqual( + self.mw.process_response(request1, response, self.spider), + response, + ) + + request2 = self.redirect_middleware.process_response( + request1, + response, + self.spider, + ) + self.assertIsInstance(request2, Request) + + self.mw.process_request(request2, self.spider) + cookies = request2.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + def test_cookie_redirect_same_domain(self): + self._test_cookie_redirect( + 'https://toscrape.com', + 'https://toscrape.com', + cookies1=True, + cookies2=True, + ) + + def test_cookie_redirect_same_domain_forcing_get(self): + self._test_cookie_redirect( + 'https://toscrape.com', + {'url': 'https://toscrape.com', 'status': 302}, + cookies1=True, + cookies2=True, + ) + + def test_cookie_redirect_different_domain(self): + self._test_cookie_redirect( + 'https://toscrape.com', + 'https://example.com', + cookies1=True, + cookies2=False, + ) + + def test_cookie_redirect_different_domain_forcing_get(self): + self._test_cookie_redirect( + 'https://toscrape.com', + {'url': 'https://example.com', 'status': 302}, + cookies1=True, + cookies2=False, + ) + + def _test_cookie_header_redirect( + self, + source, + target, + *, + cookies2, + ): + """Test the handling of a user-defined Cookie header when building a + redirect follow-up request. + + We follow RFC 6265 for cookie handling. The Cookie header can only + contain a list of key-value pairs (i.e. no additional cookie + parameters like Domain or Path). Because of that, we follow the same + rules that we would follow for the handling of the Set-Cookie response + header when the Domain is not set: the cookies must be limited to the + target URL domain (not even subdomains can receive those cookies). + + .. note:: This method tests the scenario where the cookie middleware is + disabled. Because of known issue #1992, when the cookies + middleware is enabled we do not need to be concerned about + the Cookie header getting leaked to unintended domains, + because the middleware empties the header from every request. + """ + if not isinstance(source, dict): + source = {'url': source} + if not isinstance(target, dict): + target = {'url': target} + target.setdefault('status', 301) + + request1 = Request(headers={'Cookie': b'a=b'}, **source) + + response = Response( + headers={ + 'Location': target['url'], + }, + **target, + ) + + request2 = self.redirect_middleware.process_response( + request1, + response, + self.spider, + ) + self.assertIsInstance(request2, Request) + + cookies = request2.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + def test_cookie_header_redirect_same_domain(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + 'https://toscrape.com', + cookies2=True, + ) + + def test_cookie_header_redirect_same_domain_forcing_get(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + {'url': 'https://toscrape.com', 'status': 302}, + cookies2=True, + ) + + def test_cookie_header_redirect_different_domain(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + 'https://example.com', + cookies2=False, + ) + + def test_cookie_header_redirect_different_domain_forcing_get(self): + self._test_cookie_header_redirect( + 'https://toscrape.com', + {'url': 'https://example.com', 'status': 302}, + cookies2=False, + ) + + def _test_user_set_cookie_domain_followup( + self, + url1, + url2, + domain, + *, + cookies1, + cookies2, + ): + input_cookies = [ + { + 'name': 'a', + 'value': 'b', + 'domain': domain, + } + ] + + request1 = Request(url1, cookies=input_cookies) + self.mw.process_request(request1, self.spider) + cookies = request1.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies1 else None) + + request2 = Request(url2) + self.mw.process_request(request2, self.spider) + cookies = request2.headers.get('Cookie') + self.assertEqual(cookies, b"a=b" if cookies2 else None) + + def test_user_set_cookie_domain_suffix_private(self): + self._test_user_set_cookie_domain_followup( + 'https://books.toscrape.com', + 'https://quotes.toscrape.com', + 'toscrape.com', + cookies1=True, + cookies2=True, + ) + + def test_user_set_cookie_domain_suffix_public_period(self): + self._test_user_set_cookie_domain_followup( + 'https://foo.co.uk', + 'https://bar.co.uk', + 'co.uk', + cookies1=False, + cookies2=False, + ) + + def test_user_set_cookie_domain_suffix_public_private(self): + self._test_user_set_cookie_domain_followup( + 'https://foo.blogspot.com', + 'https://bar.blogspot.com', + 'blogspot.com', + cookies1=False, + cookies2=False, + ) + + def test_user_set_cookie_domain_public_period(self): + self._test_user_set_cookie_domain_followup( + 'https://co.uk', + 'https://co.uk', + 'co.uk', + cookies1=True, + cookies2=True, + ) + + def _test_server_set_cookie_domain_followup( + self, + url1, + url2, + domain, + *, + cookies, + ): + request1 = Request(url1) + self.mw.process_request(request1, self.spider) + + input_cookies = [ + { + 'name': 'a', + 'value': 'b', + 'domain': domain, + } + ] + + headers = { + 'Set-Cookie': _cookies_to_set_cookie_list(input_cookies), + } + response = Response(url1, status=200, headers=headers) + self.assertEqual( + self.mw.process_response(request1, response, self.spider), + response, + ) + + request2 = Request(url2) + self.mw.process_request(request2, self.spider) + actual_cookies = request2.headers.get('Cookie') + self.assertEqual(actual_cookies, b"a=b" if cookies else None) + + def test_server_set_cookie_domain_suffix_private(self): + self._test_server_set_cookie_domain_followup( + 'https://books.toscrape.com', + 'https://quotes.toscrape.com', + 'toscrape.com', + cookies=True, + ) + + def test_server_set_cookie_domain_suffix_public_period(self): + self._test_server_set_cookie_domain_followup( + 'https://foo.co.uk', + 'https://bar.co.uk', + 'co.uk', + cookies=False, + ) + + def test_server_set_cookie_domain_suffix_public_private(self): + self._test_server_set_cookie_domain_followup( + 'https://foo.blogspot.com', + 'https://bar.blogspot.com', + 'blogspot.com', + cookies=False, + ) + + def test_server_set_cookie_domain_public_period(self): + self._test_server_set_cookie_domain_followup( + 'https://co.uk', + 'https://co.uk', + 'co.uk', + cookies=True, + ) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 3381632b0..0362e2018 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,13 +1,60 @@ import unittest +from w3lib.http import basic_auth_header + from scrapy.http import Request from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spiders import Spider +class TestSpiderLegacy(Spider): + http_user = 'foo' + http_pass = 'bar' + + class TestSpider(Spider): http_user = 'foo' http_pass = 'bar' + http_auth_domain = 'example.com' + + +class TestSpiderAny(Spider): + http_user = 'foo' + http_pass = 'bar' + http_auth_domain = None + + +class HttpAuthMiddlewareLegacyTest(unittest.TestCase): + + def setUp(self): + self.spider = TestSpiderLegacy('foo') + + def test_auth(self): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) + + # initial request, sets the domain and sends the header + req = Request('http://example.com/') + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + # subsequent request to the same domain, should send the header + req = Request('http://example.com/') + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + # subsequent request to a different domain, shouldn't send the header + req = Request('http://example-noauth.com/') + assert mw.process_request(req, self.spider) is None + self.assertNotIn('Authorization', req.headers) + + def test_auth_already_set(self): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) + req = Request('http://example.com/', + headers=dict(Authorization='Digest 123')) + assert mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') class HttpAuthMiddlewareTest(unittest.TestCase): @@ -20,13 +67,45 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def tearDown(self): del self.mw - def test_auth(self): - req = Request('http://scrapytest.org/') + def test_no_auth(self): + req = Request('http://example-noauth.com/') assert self.mw.process_request(req, self.spider) is None - self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') + self.assertNotIn('Authorization', req.headers) + + def test_auth_domain(self): + req = Request('http://example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + def test_auth_subdomain(self): + req = Request('http://foo.example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) def test_auth_already_set(self): - req = Request('http://scrapytest.org/', + req = Request('http://example.com/', + headers=dict(Authorization='Digest 123')) + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], b'Digest 123') + + +class HttpAuthAnyMiddlewareTest(unittest.TestCase): + + def setUp(self): + self.mw = HttpAuthMiddleware() + self.spider = TestSpiderAny('foo') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_auth(self): + req = Request('http://example.com/') + assert self.mw.process_request(req, self.spider) is None + self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar')) + + def test_auth_already_set(self): + req = Request('http://example.com/', headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers['Authorization'], b'Digest 123') diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 4c5bfc577..40e9f3a96 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,13 +1,16 @@ -from io import BytesIO -from unittest import TestCase, SkipTest -from os.path import join from gzip import GzipFile +from io import BytesIO +from os.path import join +from unittest import TestCase, SkipTest +from warnings import catch_warnings from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip +from scrapy.utils.test import get_crawler from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -32,8 +35,10 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = Spider('foo') - self.mw = HttpCompressionMiddleware() + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('scrapytest.org') + self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): if coding not in FORMAT: @@ -56,6 +61,34 @@ class HttpCompressionTest(TestCase): response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) return response + def assertStatsEqual(self, key, value): + self.assertEqual( + self.crawler.stats.get_value(key, spider=self.spider), + value, + str(self.crawler.stats.get_stats(self.spider)) + ) + + def test_setting_false_compression_enabled(self): + self.assertRaises( + NotConfigured, + HttpCompressionMiddleware.from_crawler, + get_crawler(settings_dict={'COMPRESSION_ENABLED': False}) + ) + + def test_setting_default_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler(get_crawler()), + HttpCompressionMiddleware + ) + + def test_setting_true_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler( + get_crawler(settings_dict={'COMPRESSION_ENABLED': True}) + ), + HttpCompressionMiddleware + ) + def test_process_request(self): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers @@ -72,6 +105,20 @@ class HttpCompressionTest(TestCase): assert newresponse is not response assert newresponse.body.startswith(b' meta(max_retry_times) - self.mw.max_retry_times = 5 meta_max_retry_times = 4 + middleware_max_retry_times = 5 req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) req2 = Request(self.invalid_url) - self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) - self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + settings = {'RETRY_TIMES': middleware_max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) + + self._test_retry( + req1, + DNSLookupError('foo'), + meta_max_retry_times, + spider=spider, + middleware=middleware, + ) + self._test_retry( + req2, + DNSLookupError('foo'), + middleware_max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_dont_retry(self): + max_retry_times = 4 + spider, middleware = self.get_spider_and_middleware() + meta = { + 'max_retry_times': max_retry_times, + 'dont_retry': True, + } + req = Request(self.invalid_url, meta=meta) + self._test_retry( + req, + DNSLookupError('foo'), + 0, + spider=spider, + middleware=middleware, + ) - # SETTINGS: meta(max_retry_times) = 4 - meta_max_retry_times = 4 - - req = Request(self.invalid_url, meta={ - 'max_retry_times': meta_max_retry_times, 'dont_retry': True - }) - - self._test_retry(req, DNSLookupError('foo'), 0) - - def _test_retry(self, req, exception, max_retry_times): + def _test_retry( + self, + req, + exception, + max_retry_times, + spider=None, + middleware=None, + ): + spider = spider or self.spider + middleware = middleware or self.mw for i in range(0, max_retry_times): - req = self.mw.process_exception(req, exception, self.spider) + req = middleware.process_exception(req, exception, spider) assert isinstance(req, Request) # discard it - req = self.mw.process_exception(req, exception, self.spider) + req = middleware.process_exception(req, exception, spider) self.assertEqual(req, None) +class GetRetryRequestTest(unittest.TestCase): + + def get_spider(self, settings=None): + crawler = get_crawler(Spider, settings or {}) + return crawler._create_spider('foo') + + def test_basic_usage(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -1) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_max_retries_reached(self): + request = Request('https://example.com') + spider = self.get_spider() + max_retry_times = 0 + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + self.assertEqual( + spider.crawler.stats.get_value('retry/max_reached'), + 1 + ) + failure_count = max_retry_times + 1 + expected_reason = "unspecified" + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) + ) + + def test_one_retry(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=1, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -1) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_two_retries(self): + spider = self.get_spider() + request = Request('https://example.com') + new_request = request + max_retry_times = 2 + for index in range(max_retry_times): + with LogCapture() as log: + new_request = get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = index + 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -expected_retry_times) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + value = spider.crawler.stats.get_value(stat) + self.assertEqual(value, expected_retry_times) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + with LogCapture() as log: + new_request = get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + self.assertEqual( + spider.crawler.stats.get_value('retry/max_reached'), + 1 + ) + failure_count = max_retry_times + 1 + expected_reason = "unspecified" + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) + ) + + def test_no_spider(self): + request = Request('https://example.com') + with self.assertRaises(TypeError): + get_retry_request(request) # pylint: disable=missing-kwoa + + def test_max_retry_times_setting(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request, None) + + def test_max_retry_times_meta(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) + meta = {'max_retry_times': max_retry_times} + request = Request('https://example.com', meta=meta) + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request, None) + + def test_max_retry_times_argument(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) + meta = {'max_retry_times': max_retry_times + 1} + request = Request('https://example.com', meta=meta) + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + + def test_priority_adjust_setting(self): + priority_adjust = 1 + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request.priority, priority_adjust) + + def test_priority_adjust_argument(self): + priority_adjust = 1 + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust + 1}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + priority_adjust=priority_adjust, + ) + self.assertEqual(new_request.priority, priority_adjust) + + def test_log_extra_retry_success(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture(attributes=('spider',)) as log: + get_retry_request( + request, + spider=spider, + ) + log.check_present(spider) + + def test_log_extra_retries_exceeded(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture(attributes=('spider',)) as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present(spider) + + def test_reason_string(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = 'because' + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_builtin_exception(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = NotImplementedError() + expected_reason_string = 'builtins.NotImplementedError' + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_builtin_exception_class(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = NotImplementedError + expected_reason_string = 'builtins.NotImplementedError' + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_custom_exception(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = IgnoreRequest() + expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_custom_exception_class(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = IgnoreRequest + expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_custom_logger(self): + logger = logging.getLogger("custom-logger") + request = Request("https://example.com") + spider = self.get_spider() + expected_reason = "because" + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + logger=logger, + ) + log.check_present( + ( + "custom-logger", + "DEBUG", + f"Retrying {request} (failed 1 times): {expected_reason}", + ) + ) + + def test_custom_stats_key(self): + request = Request("https://example.com") + spider = self.get_spider() + expected_reason = "because" + stats_key = "custom_retry" + get_retry_request( + request, + spider=spider, + reason=expected_reason, + stats_base_key=stats_key, + ) + for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 858138f81..1460d88eb 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -42,7 +42,7 @@ Disallow: /some/randome/page.html """.encode('utf-8') response = TextResponse('http://site.local/robots.txt', body=ROBOTS) - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -79,7 +79,7 @@ Disallow: /some/randome/page.html crawler.settings.set('ROBOTSTXT_OBEY', True) response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2') - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -102,7 +102,7 @@ Disallow: /some/randome/page.html crawler.settings.set('ROBOTSTXT_OBEY', True) response = Response('http://site.local/robots.txt') - def return_response(request, spider): + def return_response(request): deferred = Deferred() reactor.callFromThread(deferred.callback, response) return deferred @@ -122,7 +122,7 @@ Disallow: /some/randome/page.html self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') - def return_failure(request, spider): + def return_failure(request): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(err)) return deferred @@ -138,7 +138,7 @@ Disallow: /some/randome/page.html self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') - def immediate_failure(request, spider): + def immediate_failure(request): deferred = Deferred() deferred.errback(failure.Failure(err)) return deferred @@ -150,7 +150,7 @@ Disallow: /some/randome/page.html def test_ignore_robotstxt_request(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) - def ignore_request(request, spider): + def ignore_request(request): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest())) return deferred diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 1f2616e35..9e75f0a50 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,8 +1,10 @@ +from itertools import product from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.response import response_httprepr from scrapy.utils.test import get_crawler @@ -37,6 +39,23 @@ class TestDownloaderStats(TestCase): self.mw.process_response(self.req, self.res, self.spider) self.assertStatsEqual('downloader/response_count', 1) + def test_response_len(self): + body = (b'', b'not_empty') # empty/notempty body + headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'}) # 0 headers, 1h and 2h + test_responses = [ # form test responses with all combinations of body/headers + Response( + url='scrapytest.org', + status=200, + body=r[0], + headers=r[1] + ) + for r in product(body, headers) + ] + for test_response in test_responses: + self.crawler.stats.set_value('downloader/response_bytes', 0) + self.mw.process_response(self.req, test_response, self.spider) + self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response))) + def test_process_exception(self): self.mw.process_exception(self.req, MyException(), self.spider) self.assertStatsEqual('downloader/exception_count', 1) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 680bb6dc8..b7df2554a 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -15,6 +15,16 @@ from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider +def _get_dupefilter(*, crawler=None, settings=None, open=True): + if crawler is None: + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + dupefilter = scheduler.df + if open: + dupefilter.open() + return dupefilter + + class FromCrawlerRFPDupeFilter(RFPDupeFilter): @classmethod @@ -64,9 +74,7 @@ class RFPDupeFilterTest(unittest.TestCase): self.assertEqual(scheduler.df.method, 'n/a') def test_filter(self): - dupefilter = RFPDupeFilter() - dupefilter.open() - + dupefilter = _get_dupefilter() r1 = Request('http://scrapytest.org/1') r2 = Request('http://scrapytest.org/2') r3 = Request('http://scrapytest.org/2') @@ -85,7 +93,7 @@ class RFPDupeFilterTest(unittest.TestCase): path = tempfile.mkdtemp() try: - df = RFPDupeFilter(path) + df = _get_dupefilter(settings={'JOBDIR': path}, open=False) try: df.open() assert not df.request_seen(r1) @@ -93,7 +101,8 @@ class RFPDupeFilterTest(unittest.TestCase): finally: df.close('finished') - df2 = RFPDupeFilter(path) + df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False) + assert df != df2 try: df2.open() assert df2.request_seen(r1) @@ -109,26 +118,24 @@ class RFPDupeFilterTest(unittest.TestCase): output of request_seen. """ + dupefilter = _get_dupefilter() r1 = Request('http://scrapytest.org/index.html') r2 = Request('http://scrapytest.org/INDEX.html') - dupefilter = RFPDupeFilter() - dupefilter.open() - assert not dupefilter.request_seen(r1) assert not dupefilter.request_seen(r2) dupefilter.close('finished') - class CaseInsensitiveRFPDupeFilter(RFPDupeFilter): + class RequestFingerprinter: - def request_fingerprint(self, request): + def fingerprint(self, request): fp = hashlib.sha1() fp.update(to_bytes(request.url.lower())) - return fp.hexdigest() + return fp.digest() - case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter() - case_insensitive_dupefilter.open() + settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter} + case_insensitive_dupefilter = _get_dupefilter(settings=settings) assert not case_insensitive_dupefilter.request_seen(r1) assert case_insensitive_dupefilter.request_seen(r2) @@ -142,8 +149,10 @@ class RFPDupeFilterTest(unittest.TestCase): r1 = Request('http://scrapytest.org/1') path = tempfile.mkdtemp() + crawler = get_crawler(settings_dict={'JOBDIR': path}) try: - df = RFPDupeFilter(path) + scheduler = Scheduler.from_crawler(crawler) + df = scheduler.df df.open() df.request_seen(r1) df.close('finished') @@ -164,11 +173,8 @@ class RFPDupeFilterTest(unittest.TestCase): settings = {'DUPEFILTER_DEBUG': False, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(SimpleSpider, settings_dict=settings) - scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) - - dupefilter = scheduler.df - dupefilter.open() + dupefilter = _get_dupefilter(crawler=crawler) r1 = Request('http://scrapytest.org/index.html') r2 = Request('http://scrapytest.org/index.html') @@ -193,11 +199,41 @@ class RFPDupeFilterTest(unittest.TestCase): settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(SimpleSpider, settings_dict=settings) - scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) - - dupefilter = scheduler.df - dupefilter.open() + dupefilter = _get_dupefilter(crawler=crawler) + + r1 = Request('http://scrapytest.org/index.html') + r2 = Request('http://scrapytest.org/index.html', + headers={'Referer': 'http://scrapytest.org/INDEX.html'}) + + dupefilter.log(r1, spider) + dupefilter.log(r2, spider) + + assert crawler.stats.get_value('dupefilter/filtered') == 2 + log.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: (referer: None)' + ) + ) + log.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: ' + ' (referer: http://scrapytest.org/INDEX.html)' + ) + ) + + dupefilter.close('finished') + + def test_log_debug_default_dupefilter(self): + with LogCapture() as log: + settings = {'DUPEFILTER_DEBUG': True} + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) r1 = Request('http://scrapytest.org/index.html') r2 = Request('http://scrapytest.org/index.html', diff --git a/tests/test_engine.py b/tests/test_engine.py index 3629aa1aa..fa7d0c8d4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -13,20 +13,20 @@ module with the ``runserver`` argument:: import os import re import sys +import warnings from collections import defaultdict from urllib.parse import urlparse import attr from itemadapter import ItemAdapter from pydispatch import dispatcher -from testfixtures import LogCapture from twisted.internet import defer, reactor from twisted.trial import unittest from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import StopDownload +from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -58,7 +58,7 @@ class TestSpider(Spider): name_re = re.compile(r"

(.*?)

", re.M) price_re = re.compile(r">Price: \$(.*?)<", re.M) - item_cls = TestItem + item_cls: type = TestItem def parse(self, response): xlink = LinkExtractor() @@ -68,15 +68,15 @@ class TestSpider(Spider): yield Request(url=link.url, callback=self.parse_item) def parse_item(self, response): - item = self.item_cls() + adapter = ItemAdapter(self.item_cls()) m = self.name_re.search(response.text) if m: - item['name'] = m.group(1) - item['url'] = response.url + adapter['name'] = m.group(1) + adapter['url'] = response.url m = self.price_re.search(response.text) if m: - item['price'] = m.group(1) - return item + adapter['price'] = m.group(1) + return adapter.item class TestDupeFilterSpider(TestSpider): @@ -89,7 +89,7 @@ class DictItemsSpider(TestSpider): class AttrsItemsSpider(TestSpider): - item_class = AttrsItem + item_cls = AttrsItem try: @@ -99,14 +99,10 @@ except ImportError: else: TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - class DataClassItemsSpider(DictItemsSpider): + class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef] def parse_item(self, response): item = super().parse_item(response) - return TestDataClass( - name=item.get('name'), - url=item.get('url'), - price=item.get('price'), - ) + return TestDataClass(**item) class ItemZeroDivisionErrorSpider(TestSpider): @@ -117,6 +113,18 @@ class ItemZeroDivisionErrorSpider(TestSpider): } +class ChangeCloseReasonSpider(TestSpider): + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(*args, **kwargs) + spider._set_crawler(crawler) + crawler.signals.connect(spider.spider_idle, signals.spider_idle) + return spider + + def spider_idle(self): + raise CloseSpider(reason="custom_reason") + + def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) @@ -143,7 +151,8 @@ class CrawlerRun: self.reqreached = [] self.itemerror = [] self.itemresp = [] - self.bytes = defaultdict(lambda: list()) + self.headers = {} + self.bytes = defaultdict(list) self.signals_caught = {} self.spider_class = spider_class @@ -165,6 +174,7 @@ class CrawlerRun: self.crawler = get_crawler(self.spider_class) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.item_error, signals.item_error) + self.crawler.signals.connect(self.headers_received, signals.headers_received) self.crawler.signals.connect(self.bytes_received, signals.bytes_received) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) @@ -183,6 +193,7 @@ class CrawlerRun: if not name.startswith('_'): disconnect_all(signal) self.deferred.callback(None) + return self.crawler.stop() def geturl(self, path): return f"http://localhost:{self.portno}{path}" @@ -197,6 +208,9 @@ class CrawlerRun: def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) + def headers_received(self, headers, body_length, request, spider): + self.headers[request] = headers + def bytes_received(self, data, request, spider): self.bytes[request].append(data) @@ -220,18 +234,7 @@ class CrawlerRun: self.signals_caught[sig] = signalargs -class StopDownloadCrawlerRun(CrawlerRun): - """ - Make sure raising the StopDownload exception stops the download of the response body - """ - - def bytes_received(self, data, request, spider): - super().bytes_received(data, request, spider) - raise StopDownload(fail=False) - - class EngineTest(unittest.TestCase): - @defer.inlineCallbacks def test_crawler(self): @@ -241,8 +244,8 @@ class EngineTest(unittest.TestCase): self.run = CrawlerRun(spider) yield self.run.run() self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=9) - self._assert_downloaded_responses() + self._assert_scheduled_requests(count=9) + self._assert_downloaded_responses(count=9) self._assert_scraped_items() self._assert_signals_caught() self._assert_bytes_received() @@ -251,7 +254,7 @@ class EngineTest(unittest.TestCase): def test_crawler_dupefilter(self): self.run = CrawlerRun(TestDupeFilterSpider) yield self.run.run() - self._assert_scheduled_requests(urls_to_visit=8) + self._assert_scheduled_requests(count=8) self._assert_dropped_requests() @defer.inlineCallbacks @@ -260,6 +263,13 @@ class EngineTest(unittest.TestCase): yield self.run.run() self._assert_items_error() + @defer.inlineCallbacks + def test_crawler_change_close_reason_on_idle(self): + self.run = CrawlerRun(ChangeCloseReasonSpider) + yield self.run.run() + self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'}, + self.run.signals_caught[signals.spider_closed]) + def _assert_visited_urls(self): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] @@ -267,8 +277,8 @@ class EngineTest(unittest.TestCase): urls_expected = {self.run.geturl(p) for p in must_be_visited} assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" - def _assert_scheduled_requests(self, urls_to_visit=None): - self.assertEqual(urls_to_visit, len(self.run.reqplug)) + def _assert_scheduled_requests(self, count=None): + self.assertEqual(count, len(self.run.reqplug)) paths_expected = ['/item999.html', '/item2.html', '/item1.html'] @@ -286,10 +296,10 @@ class EngineTest(unittest.TestCase): def _assert_dropped_requests(self): self.assertEqual(len(self.run.reqdropped), 1) - def _assert_downloaded_responses(self): + def _assert_downloaded_responses(self, count): # response tests - self.assertEqual(9, len(self.run.respplug)) - self.assertEqual(9, len(self.run.reqreached)) + self.assertEqual(count, len(self.run.respplug)) + self.assertEqual(count, len(self.run.reqreached)) for response, _ in self.run.respplug: if self.run.getpath(response.url) == '/item999.html': @@ -323,6 +333,13 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) + def _assert_headers_received(self): + for headers in self.run.headers.values(): + self.assertIn(b"Server", headers) + self.assertIn(b"TwistedWeb", headers[b"Server"]) + self.assertIn(b"Date", headers) + self.assertIn(b"Content-Type", headers) + def _assert_bytes_received(self): self.assertEqual(9, len(self.run.bytes)) for request, data in self.run.bytes.items(): @@ -371,6 +388,7 @@ class EngineTest(unittest.TestCase): assert signals.spider_opened in self.run.signals_caught assert signals.spider_idle in self.run.signals_caught assert signals.spider_closed in self.run.signals_caught + assert signals.headers_received in self.run.signals_caught self.assertEqual({'spider': self.run.spider}, self.run.signals_caught[signals.spider_opened]) @@ -385,64 +403,104 @@ class EngineTest(unittest.TestCase): yield e.close() @defer.inlineCallbacks - def test_close_spiders_downloader(self): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - self.assertEqual(len(e.open_spiders), 1) - yield e.close() - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_close_engine_spiders_downloader(self): + def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() - self.assertTrue(e.running) - yield e.close() - self.assertFalse(e.running) - self.assertEqual(len(e.open_spiders), 0) - - -class StopDownloadEngineTest(EngineTest): + yield self.assertFailure(e.start(), RuntimeError).addBoth( + lambda exc: self.assertEqual(str(exc), "Engine already running") + ) + yield e.stop() @defer.inlineCallbacks - def test_crawler(self): - for spider in TestSpider, DictItemsSpider: - self.run = StopDownloadCrawlerRun(spider) - with LogCapture() as log: - yield self.run.run() - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped for " - "from signal handler" - " StopDownloadCrawlerRun.bytes_received")) - self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=9) - self._assert_downloaded_responses() - self._assert_signals_caught() - self._assert_bytes_received() + def test_close_spiders_downloader(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + yield e.open_spider(TestSpider(), []) + self.assertEqual(len(e.open_spiders), 1) + yield e.close() + self.assertEqual(len(e.open_spiders), 0) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + ) - def _assert_bytes_received(self): - self.assertEqual(9, len(self.run.bytes)) - for request, data in self.run.bytes.items(): - joined_data = b"".join(data) - self.assertTrue(len(data) == 1) # signal was fired only once - if self.run.getpath(request.url) == "/numbers": - # Received bytes are not the complete response. The exact amount depends - # on the buffer size, which can vary, so we only check that the amount - # of received bytes is strictly less than the full response. - numbers = [str(x).encode("utf8") for x in range(2**18)] - self.assertTrue(len(joined_data) < len(b"".join(numbers))) + @defer.inlineCallbacks + def test_close_engine_spiders_downloader(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + yield e.open_spider(TestSpider(), []) + e.start() + self.assertTrue(e.running) + yield e.close() + self.assertFalse(e.running) + self.assertEqual(len(e.open_spiders), 0) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", + ) + + @defer.inlineCallbacks + def test_crawl_deprecated_spider_arg(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.crawl(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", + ) + + @defer.inlineCallbacks + def test_download_deprecated_spider_arg(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.download(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "Passing a 'spider' argument to ExecutionEngine.download is deprecated", + ) + + @defer.inlineCallbacks + def test_deprecated_schedule(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + spider = TestSpider() + yield e.open_spider(spider, []) + e.start() + e.schedule(Request("data:,"), spider) + yield e.close() + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual( + str(warning_list[0].message), + "ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead", + ) + + @defer.inlineCallbacks + def test_deprecated_has_capacity(self): + with warnings.catch_warnings(record=True) as warning_list: + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) + self.assertTrue(e.has_capacity()) + spider = TestSpider() + yield e.open_spider(spider, []) + self.assertFalse(e.has_capacity()) + e.start() + yield e.close() + self.assertTrue(e.has_capacity()) + self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) + self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated") if __name__ == "__main__": diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py new file mode 100644 index 000000000..0ba69e096 --- /dev/null +++ b/tests/test_engine_stop_download_bytes.py @@ -0,0 +1,60 @@ +from testfixtures import LogCapture +from twisted.internet import defer + +from scrapy.exceptions import StopDownload + +from tests.test_engine import ( + AttrsItemsSpider, + DataClassItemsSpider, + DictItemsSpider, + TestSpider, + CrawlerRun, + EngineTest, +) + + +class BytesReceivedCrawlerRun(CrawlerRun): + def bytes_received(self, data, request, spider): + super().bytes_received(data, request, spider) + raise StopDownload(fail=False) + + +class BytesReceivedEngineTest(EngineTest): + @defer.inlineCallbacks + def test_crawler(self): + for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + if spider is None: + continue + self.run = BytesReceivedCrawlerRun(spider) + with LogCapture() as log: + yield self.run.run() + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for " + "from signal handler BytesReceivedCrawlerRun.bytes_received")) + self._assert_visited_urls() + self._assert_scheduled_requests(count=9) + self._assert_downloaded_responses(count=9) + self._assert_signals_caught() + self._assert_headers_received() + self._assert_bytes_received() + + def _assert_bytes_received(self): + self.assertEqual(9, len(self.run.bytes)) + for request, data in self.run.bytes.items(): + joined_data = b"".join(data) + self.assertTrue(len(data) == 1) # signal was fired only once + if self.run.getpath(request.url) == "/numbers": + # Received bytes are not the complete response. The exact amount depends + # on the buffer size, which can vary, so we only check that the amount + # of received bytes is strictly less than the full response. + numbers = [str(x).encode("utf8") for x in range(2**18)] + self.assertTrue(len(joined_data) < len(b"".join(numbers))) diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py new file mode 100644 index 000000000..fad6643ad --- /dev/null +++ b/tests/test_engine_stop_download_headers.py @@ -0,0 +1,56 @@ +from testfixtures import LogCapture +from twisted.internet import defer + +from scrapy.exceptions import StopDownload + +from tests.test_engine import ( + AttrsItemsSpider, + DataClassItemsSpider, + DictItemsSpider, + TestSpider, + CrawlerRun, + EngineTest, +) + + +class HeadersReceivedCrawlerRun(CrawlerRun): + def headers_received(self, headers, body_length, request, spider): + super().headers_received(headers, body_length, request, spider) + raise StopDownload(fail=False) + + +class HeadersReceivedEngineTest(EngineTest): + @defer.inlineCallbacks + def test_crawler(self): + for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + if spider is None: + continue + self.run = HeadersReceivedCrawlerRun(spider) + with LogCapture() as log: + yield self.run.run() + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from" + " signal handler HeadersReceivedCrawlerRun.headers_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from signal" + " handler HeadersReceivedCrawlerRun.headers_received")) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + f"Download stopped for from" + " signal handler HeadersReceivedCrawlerRun.headers_received")) + self._assert_visited_urls() + self._assert_downloaded_responses(count=6) + self._assert_signals_caught() + self._assert_bytes_received() + self._assert_headers_received() + + def _assert_bytes_received(self): + self.assertEqual(0, len(self.run.bytes)) + + def _assert_visited_urls(self): + must_be_visited = ["/", "/redirect", "/redirected"] + urls_visited = {rp[0].url for rp in self.run.respplug} + urls_expected = {self.run.geturl(p) for p in must_be_visited} + assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" diff --git a/tests/test_exporters.py b/tests/test_exporters.py index d15b2ad27..6ba7428f6 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -7,12 +7,14 @@ import unittest from collections import OrderedDict from io import BytesIO from datetime import datetime +from warnings import catch_warnings, filterwarnings import lxml.etree from itemadapter import ItemAdapter from scrapy.item import Item, Field from scrapy.utils.python import to_unicode +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, @@ -181,10 +183,12 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_binary(self): - exporter = PythonItemExporter(binary=True) - value = self.item_class(name='John\xa3', age='22') - expected = {b'name': b'John\xc2\xa3', b'age': b'22'} - self.assertEqual(expected, exporter.export_item(value)) + with catch_warnings(): + filterwarnings('ignore', category=ScrapyDeprecationWarning) + exporter = PythonItemExporter(binary=True) + value = self.item_class(name='John\xa3', age='22') + expected = {b'name': b'John\xc2\xa3', b'age': b'22'} + self.assertEqual(expected, exporter.export_item(value)) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -369,14 +373,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item=dict(text=u'W\u0275\u200Brd'), + item=dict(text='W\u0275\u200Brd'), expected=None, encoding='windows-1251', ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text=u'W\u0275\u200Brd'), + item=dict(text='W\u0275\u200Brd'), include_headers_line=False, expected='Wɵ​rd\r\n', encoding='windows-1251', diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 25b1ccf63..83aabbdc7 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,5 +1,8 @@ +import bz2 import csv +import gzip import json +import lzma import os import random import shutil @@ -246,7 +249,8 @@ class S3FeedStorageTest(unittest.TestCase): def test_parse_credentials(self): skip_if_no_boto() aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', - 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} + 'AWS_SECRET_ACCESS_KEY': 'settings_secret', + 'AWS_SESSION_TOKEN': 'settings_token'} crawler = get_crawler(settings_dict=aws_credentials) # Instantiate with crawler storage = S3FeedStorage.from_crawler( @@ -255,12 +259,15 @@ class S3FeedStorageTest(unittest.TestCase): ) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') + self.assertEqual(storage.session_token, 'settings_token') # Instantiate directly storage = S3FeedStorage('s3://mybucket/export.csv', aws_credentials['AWS_ACCESS_KEY_ID'], - aws_credentials['AWS_SECRET_ACCESS_KEY']) + aws_credentials['AWS_SECRET_ACCESS_KEY'], + session_token=aws_credentials['AWS_SESSION_TOKEN']) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') + self.assertEqual(storage.session_token, 'settings_token') # URI priority > settings priority storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv', aws_credentials['AWS_ACCESS_KEY_ID'], @@ -328,6 +335,17 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def test_init_with_endpoint_url(self): + storage = S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + endpoint_url='https://example.com' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, 'https://example.com') + def test_from_crawler_without_acl(self): settings = { 'AWS_ACCESS_KEY_ID': 'access_key', @@ -342,6 +360,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, None) + def test_without_endpoint_url(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv', + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, None) + def test_from_crawler_with_acl(self): settings = { 'AWS_ACCESS_KEY_ID': 'access_key', @@ -357,6 +389,21 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, 'secret_key') self.assertEqual(storage.acl, 'custom-acl') + def test_from_crawler_with_endpoint_url(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'AWS_ENDPOINT_URL': 'https://example.com', + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv' + ) + self.assertEqual(storage.access_key, 'access_key') + self.assertEqual(storage.secret_key, 'secret_key') + self.assertEqual(storage.endpoint_url, 'https://example.com') + @defer.inlineCallbacks def test_store_botocore_without_acl(self): skip_if_no_boto() @@ -517,7 +564,7 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): class DummyBlockingFeedStorage(BlockingFeedStorage): - def __init__(self, uri): + def __init__(self, uri, *args, feed_options=None): self.path = file_uri_to_path(uri) def _store_in_thread(self, file): @@ -543,7 +590,7 @@ class LogOnStoreFileStorage: It can be used to make sure `store` method is invoked. """ - def __init__(self, uri): + def __init__(self, uri, feed_options=None): self.path = file_uri_to_path(uri) self.logger = getLogger() @@ -563,6 +610,10 @@ class FeedExportTestBase(ABC, unittest.TestCase): egg = scrapy.Field() baz = scrapy.Field() + class MyItem2(scrapy.Item): + foo = scrapy.Field() + hello = scrapy.Field() + def _random_temp_filename(self, inter_dir=''): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) @@ -890,13 +941,9 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_item_classes(self): - class MyItem2(scrapy.Item): - foo = scrapy.Field() - hello = scrapy.Field() - items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - MyItem2({'hello': 'world2', 'foo': 'bar2'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}), {'hello': 'world4', 'egg': 'spam4'}, ] @@ -984,6 +1031,114 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExported(items, list(header.values()), rows, settings=settings) + @defer.inlineCallbacks + def test_export_based_on_item_classes(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), + {'hello': 'world3', 'egg': 'spam3'}, + ] + + formats = { + 'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n', + 'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]', + 'jsonlines': ( + b'{"foo": "bar1", "egg": "spam1"}\n' + b'{"hello": "world2", "foo": "bar2"}\n' + ), + 'xml': ( + b'\n\n' + b'bar1spam1\n' + b'world2bar2\nworld3' + b'spam3\n' + ), + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'csv', + 'item_classes': [self.MyItem], + }, + self._random_temp_filename(): { + 'format': 'json', + 'item_classes': [self.MyItem2], + }, + self._random_temp_filename(): { + 'format': 'jsonlines', + 'item_classes': [self.MyItem, self.MyItem2], + }, + self._random_temp_filename(): { + 'format': 'xml', + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) + + @defer.inlineCallbacks + def test_export_based_on_custom_filters(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), + {'hello': 'world3', 'egg': 'spam3'}, + ] + + MyItem = self.MyItem + + class CustomFilter1: + def __init__(self, feed_options): + pass + + def accepts(self, item): + return isinstance(item, MyItem) + + class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): + def accepts(self, item): + if 'foo' not in item.fields: + return False + return True + + class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): + def accepts(self, item): + if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1": + return True + return False + + formats = { + 'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]', + 'xml': ( + b'\n\n' + b'bar1spam1\n' + b'world2bar2\n' + ), + 'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n', + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'json', + 'item_filter': CustomFilter1, + }, + self._random_temp_filename(): { + 'format': 'xml', + 'item_filter': CustomFilter2, + }, + self._random_temp_filename(): { + 'format': 'jsonlines', + 'item_classes': [self.MyItem, self.MyItem2], + 'item_filter': CustomFilter3, + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) + @defer.inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as @@ -1376,6 +1531,499 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(row['expected'], data[feed_options['format']]) +class FeedPostProcessedExportsTest(FeedExportTestBase): + __test__ = True + + items = [{'foo': 'bar'}] + expected = b'foo\r\nbar\r\n' + + class MyPlugin1: + def __init__(self, file, feed_options): + self.file = file + self.feed_options = feed_options + self.char = self.feed_options.get('plugin1_char', b'') + + def write(self, data): + written_count = self.file.write(data) + written_count += self.file.write(self.char) + return written_count + + def close(self): + self.file.close() + + def _named_tempfile(self, name): + return os.path.join(self.temp_dir, name) + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings): + """ Run spider with specified settings; return exported data with filename. """ + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() + } + + content = {} + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + spider_cls.start_urls = [s.url('/')] + yield runner.crawl(spider_cls) + + for file_path, feed_options in FEEDS.items(): + if not os.path.exists(str(file_path)): + continue + + with open(str(file_path), 'rb') as f: + content[str(file_path)] = f.read() + + finally: + for file_path in FEEDS.keys(): + if not os.path.exists(str(file_path)): + continue + + os.remove(str(file_path)) + + return content + + def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=''): + data_stream = BytesIO() + gzipf = gzip.GzipFile(fileobj=data_stream, filename=filename, mtime=mtime, + compresslevel=compresslevel, mode="wb") + gzipf.write(data) + gzipf.close() + data_stream.seek(0) + return data_stream.read() + + @defer.inlineCallbacks + def test_gzip_plugin(self): + + filename = self._named_tempfile('gzip_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + gzip.decompress(data[filename]) + except OSError: + self.fail("Received invalid gzip data.") + + @defer.inlineCallbacks + def test_gzip_plugin_compresslevel(self): + + filename_to_compressed = { + self._named_tempfile('compresslevel_0'): self.get_gzip_compressed(self.expected, compresslevel=0), + self._named_tempfile('compresslevel_9'): self.get_gzip_compressed(self.expected, compresslevel=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('compresslevel_0'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 0, + 'gzip_mtime': 0, + 'gzip_filename': "", + }, + self._named_tempfile('compresslevel_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_compresslevel': 9, + 'gzip_mtime': 0, + 'gzip_filename': "", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_gzip_plugin_mtime(self): + filename_to_compressed = { + self._named_tempfile('mtime_123'): self.get_gzip_compressed(self.expected, mtime=123), + self._named_tempfile('mtime_123456789'): self.get_gzip_compressed(self.expected, mtime=123456789), + } + + settings = { + 'FEEDS': { + self._named_tempfile('mtime_123'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 123, + 'gzip_filename': "", + }, + self._named_tempfile('mtime_123456789'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 123456789, + 'gzip_filename': "", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_gzip_plugin_filename(self): + filename_to_compressed = { + self._named_tempfile('filename_FILE1'): self.get_gzip_compressed(self.expected, filename="FILE1"), + self._named_tempfile('filename_FILE2'): self.get_gzip_compressed(self.expected, filename="FILE2"), + } + + settings = { + 'FEEDS': { + self._named_tempfile('filename_FILE1'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 0, + 'gzip_filename': "FILE1", + }, + self._named_tempfile('filename_FILE2'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.GzipPlugin'], + 'gzip_mtime': 0, + 'gzip_filename': "FILE2", + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin(self): + + filename = self._named_tempfile('lzma_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + lzma.decompress(data[filename]) + except lzma.LZMAError: + self.fail("Received invalid lzma data.") + + @defer.inlineCallbacks + def test_lzma_plugin_format(self): + + filename_to_compressed = { + self._named_tempfile('format_FORMAT_XZ'): lzma.compress(self.expected, format=lzma.FORMAT_XZ), + self._named_tempfile('format_FORMAT_ALONE'): lzma.compress(self.expected, format=lzma.FORMAT_ALONE), + } + + settings = { + 'FEEDS': { + self._named_tempfile('format_FORMAT_XZ'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_format': lzma.FORMAT_XZ, + }, + self._named_tempfile('format_FORMAT_ALONE'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_format': lzma.FORMAT_ALONE, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_check(self): + + filename_to_compressed = { + self._named_tempfile('check_CHECK_NONE'): lzma.compress(self.expected, check=lzma.CHECK_NONE), + self._named_tempfile('check_CHECK_CRC256'): lzma.compress(self.expected, check=lzma.CHECK_SHA256), + } + + settings = { + 'FEEDS': { + self._named_tempfile('check_CHECK_NONE'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_check': lzma.CHECK_NONE, + }, + self._named_tempfile('check_CHECK_CRC256'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_check': lzma.CHECK_SHA256, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_preset(self): + + filename_to_compressed = { + self._named_tempfile('preset_PRESET_0'): lzma.compress(self.expected, preset=0), + self._named_tempfile('preset_PRESET_9'): lzma.compress(self.expected, preset=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('preset_PRESET_0'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_preset': 0, + }, + self._named_tempfile('preset_PRESET_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_preset': 9, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_lzma_plugin_filters(self): + import sys + if "PyPy" in sys.version: + # https://foss.heptapod.net/pypy/pypy/-/issues/3527 + raise unittest.SkipTest("lzma filters doesn't work in PyPy") + + filters = [{'id': lzma.FILTER_LZMA2}] + compressed = lzma.compress(self.expected, filters=filters) + filename = self._named_tempfile('filters') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.LZMAPlugin'], + 'lzma_filters': filters, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(compressed, data[filename]) + result = lzma.decompress(data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_bz2_plugin(self): + + filename = self._named_tempfile('bz2_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + try: + bz2.decompress(data[filename]) + except OSError: + self.fail("Received invalid bz2 data.") + + @defer.inlineCallbacks + def test_bz2_plugin_compresslevel(self): + + filename_to_compressed = { + self._named_tempfile('compresslevel_1'): bz2.compress(self.expected, compresslevel=1), + self._named_tempfile('compresslevel_9'): bz2.compress(self.expected, compresslevel=9), + } + + settings = { + 'FEEDS': { + self._named_tempfile('compresslevel_1'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + 'bz2_compresslevel': 1, + }, + self._named_tempfile('compresslevel_9'): { + 'format': 'csv', + 'postprocessing': ['scrapy.extensions.postprocessing.Bz2Plugin'], + 'bz2_compresslevel': 9, + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = bz2.decompress(data[filename]) + self.assertEqual(compressed, data[filename]) + self.assertEqual(self.expected, result) + + @defer.inlineCallbacks + def test_custom_plugin(self): + filename = self._named_tempfile('csv_file') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(self.expected, data[filename]) + + @defer.inlineCallbacks + def test_custom_plugin_with_parameter(self): + + expected = b'foo\r\n\nbar\r\n\n' + filename = self._named_tempfile('newline') + + settings = { + 'FEEDS': { + filename: { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + 'plugin1_char': b'\n' + }, + }, + } + + data = yield self.exported_data(self.items, settings) + self.assertEqual(expected, data[filename]) + + @defer.inlineCallbacks + def test_custom_plugin_with_compression(self): + + expected = b'foo\r\n\nbar\r\n\n' + + filename_to_decompressor = { + self._named_tempfile('bz2'): bz2.decompress, + self._named_tempfile('lzma'): lzma.decompress, + self._named_tempfile('gzip'): gzip.decompress, + } + + settings = { + 'FEEDS': { + self._named_tempfile('bz2'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.Bz2Plugin'], + 'plugin1_char': b'\n', + }, + self._named_tempfile('lzma'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.LZMAPlugin'], + 'plugin1_char': b'\n', + }, + self._named_tempfile('gzip'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], + 'plugin1_char': b'\n', + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, decompressor in filename_to_decompressor.items(): + result = decompressor(data[filename]) + self.assertEqual(expected, result) + + @defer.inlineCallbacks + def test_exports_compatibility_with_postproc(self): + import marshal + import pickle + filename_to_expected = { + self._named_tempfile('csv'): b'foo\r\nbar\r\n', + self._named_tempfile('json'): b'[\n{"foo": "bar"}\n]', + self._named_tempfile('jsonlines'): b'{"foo": "bar"}\n', + self._named_tempfile('xml'): b'\n' + b'\nbar\n', + } + + settings = { + 'FEEDS': { + self._named_tempfile('csv'): { + 'format': 'csv', + 'postprocessing': [self.MyPlugin1], + # empty plugin to activate postprocessing.PostProcessingManager + }, + self._named_tempfile('json'): { + 'format': 'json', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('jsonlines'): { + 'format': 'jsonlines', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('xml'): { + 'format': 'xml', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('marshal'): { + 'format': 'marshal', + 'postprocessing': [self.MyPlugin1], + }, + self._named_tempfile('pickle'): { + 'format': 'pickle', + 'postprocessing': [self.MyPlugin1], + }, + }, + } + + data = yield self.exported_data(self.items, settings) + + for filename, result in data.items(): + if 'pickle' in filename: + expected, result = self.items[0], pickle.loads(result) + elif 'marshal' in filename: + expected, result = self.items[0], marshal.loads(result) + else: + expected = filename_to_expected[filename] + self.assertEqual(expected, result) + + class BatchDeliveriesTest(FeedExportTestBase): __test__ = True _file_mark = '_%(batch_time)s_#%(batch_id)02d_' @@ -1835,14 +2483,15 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): maxDiff = None def test_init(self): - settings_dict = { - 'FEED_URI': 'file:///tmp/foobar', - 'FEED_STORAGES': { - 'file': FileFeedStorageWithoutFeedOptions - }, - } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with tempfile.NamedTemporaryFile() as temp: + settings_dict = { + 'FEED_URI': f'file:///{temp.name}', + 'FEED_STORAGES': { + 'file': FileFeedStorageWithoutFeedOptions + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") with warnings.catch_warnings(record=True) as w: feed_exporter.open_spider(spider) @@ -1864,8 +2513,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl): - super().__init__(uri, access_key, secret_key, acl) + def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs): + super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs) class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): @@ -2014,3 +2663,166 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): ), ) ) + + +class URIParamsTest: + + spider_name = "uri_params_spider" + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + raise NotImplementedError + + def test_default(self): + settings = self.build_settings( + uri='file:///tmp/%(name)s', + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_none(self): + def uri_params(params, spider): + pass + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual( + messages, + ( + ( + 'Modifying the params dictionary in-place in the ' + 'function defined in the FEED_URI_PARAMS setting or ' + 'in the uri_params key of the FEEDS setting is ' + 'deprecated. The function must return a new ' + 'dictionary instead.' + ), + ) + ) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_empty_dict(self): + def uri_params(params, spider): + return {} + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + with self.assertRaises(KeyError): + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + def test_params_as_is(self): + def uri_params(params, spider): + return params + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_custom_param(self): + def uri_params(params, spider): + return {**params, 'foo': self.spider_name} + + settings = self.build_settings( + uri='file:///tmp/%(foo)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + +class URIParamsSettingTest(URIParamsTest, unittest.TestCase): + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + extra_settings = {} + if uri_params: + extra_settings['FEED_URI_PARAMS'] = uri_params + return { + 'FEED_URI': uri, + **extra_settings, + } + + +class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + options = { + 'format': 'jl', + } + if uri_params: + options['uri_params'] = uri_params + return { + 'FEEDS': { + uri: options, + }, + } diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py new file mode 100644 index 000000000..49c83132f --- /dev/null +++ b/tests/test_http2_client_protocol.py @@ -0,0 +1,669 @@ +import json +import os +import random +import re +import shutil +import string +from ipaddress import IPv4Address +from unittest import mock, skipIf +from urllib.parse import urlencode + +from twisted.internet import reactor +from twisted.internet.defer import CancelledError, Deferred, DeferredList, inlineCallbacks +from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint +from twisted.internet.error import TimeoutError +from twisted.internet.ssl import optionsForClientTLS, PrivateCertificate, Certificate +from twisted.python.failure import Failure +from twisted.trial.unittest import TestCase +from twisted.web.client import ResponseFailed, URI +from twisted.web.http import H2_ENABLED, Request as TxRequest +from twisted.web.server import Site, NOT_DONE_YET +from twisted.web.static import File + +from scrapy.http import Request, Response, JsonRequest +from scrapy.settings import Settings +from scrapy.spiders import Spider +from tests.mockserver import ssl_context_factory, LeafResource, Status + + +def generate_random_string(size): + return ''.join(random.choices( + string.ascii_uppercase + string.digits, + k=size + )) + + +def make_html_body(val): + response = f''' +

Hello from HTTP2

+

{val}

+''' + return bytes(response, 'utf-8') + + +class DummySpider(Spider): + name = 'dummy' + start_urls: list = [] + + def parse(self, response): + print(response) + + +class Data: + SMALL_SIZE = 1024 # 1 KB + LARGE_SIZE = 1024 ** 2 # 1 MB + + STR_SMALL = generate_random_string(SMALL_SIZE) + STR_LARGE = generate_random_string(LARGE_SIZE) + + EXTRA_SMALL = generate_random_string(1024 * 15) + EXTRA_LARGE = generate_random_string((1024 ** 2) * 15) + + HTML_SMALL = make_html_body(STR_SMALL) + HTML_LARGE = make_html_body(STR_LARGE) + + JSON_SMALL = {'data': STR_SMALL} + JSON_LARGE = {'data': STR_LARGE} + + DATALOSS = b'Dataloss Content' + NO_CONTENT_LENGTH = b'This response do not have any content-length header' + + +class GetDataHtmlSmall(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'text/html; charset=UTF-8') + return Data.HTML_SMALL + + +class GetDataHtmlLarge(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'text/html; charset=UTF-8') + return Data.HTML_LARGE + + +class PostDataJsonMixin: + @staticmethod + def make_response(request: TxRequest, extra_data: str): + response = { + 'request-headers': {}, + 'request-body': json.loads(request.content.read()), + 'extra-data': extra_data + } + for k, v in request.requestHeaders.getAllRawHeaders(): + response['request-headers'][str(k, 'utf-8')] = str(v[0], 'utf-8') + + response_bytes = bytes(json.dumps(response), 'utf-8') + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') + return response_bytes + + +class PostDataJsonSmall(LeafResource, PostDataJsonMixin): + def render_POST(self, request: TxRequest): + return self.make_response(request, Data.EXTRA_SMALL) + + +class PostDataJsonLarge(LeafResource, PostDataJsonMixin): + def render_POST(self, request: TxRequest): + return self.make_response(request, Data.EXTRA_LARGE) + + +class Dataloss(LeafResource): + + def render_GET(self, request: TxRequest): + request.setHeader(b"Content-Length", b"1024") + self.deferRequest(request, 0, self._delayed_render, request) + return NOT_DONE_YET + + @staticmethod + def _delayed_render(request: TxRequest): + request.write(Data.DATALOSS) + request.finish() + + +class NoContentLengthHeader(LeafResource): + def render_GET(self, request: TxRequest): + request.requestHeaders.removeHeader('Content-Length') + self.deferRequest(request, 0, self._delayed_render, request) + return NOT_DONE_YET + + @staticmethod + def _delayed_render(request: TxRequest): + request.write(Data.NO_CONTENT_LENGTH) + request.finish() + + +class TimeoutResponse(LeafResource): + def render_GET(self, request: TxRequest): + return NOT_DONE_YET + + +class QueryParams(LeafResource): + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') + + query_params = {} + for k, v in request.args.items(): + query_params[str(k, 'utf-8')] = str(v[0], 'utf-8') + + return bytes(json.dumps(query_params), 'utf-8') + + +class RequestHeaders(LeafResource): + """Sends all the headers received as a response""" + + def render_GET(self, request: TxRequest): + request.setHeader('Content-Type', 'application/json; charset=UTF-8') + request.setHeader('Content-Encoding', 'UTF-8') + headers = {} + for k, v in request.requestHeaders.getAllRawHeaders(): + headers[str(k, 'utf-8')] = str(v[0], 'utf-8') + + return bytes(json.dumps(headers), 'utf-8') + + +def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: + with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: + pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) + + return PrivateCertificate.loadPEM(pem) + + +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") +class Https2ClientProtocolTestCase(TestCase): + scheme = 'https' + key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key') + certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt') + + def _init_resource(self): + self.temp_directory = self.mktemp() + os.mkdir(self.temp_directory) + r = File(self.temp_directory) + r.putChild(b'get-data-html-small', GetDataHtmlSmall()) + r.putChild(b'get-data-html-large', GetDataHtmlLarge()) + + r.putChild(b'post-data-json-small', PostDataJsonSmall()) + r.putChild(b'post-data-json-large', PostDataJsonLarge()) + + r.putChild(b'dataloss', Dataloss()) + r.putChild(b'no-content-length-header', NoContentLengthHeader()) + r.putChild(b'status', Status()) + r.putChild(b'query-params', QueryParams()) + r.putChild(b'timeout', TimeoutResponse()) + r.putChild(b'request-headers', RequestHeaders()) + return r + + @inlineCallbacks + def setUp(self): + # Initialize resource tree + root = self._init_resource() + self.site = Site(root, timeout=None) + + # Start server for testing + self.hostname = 'localhost' + context_factory = ssl_context_factory(self.key_file, self.certificate_file) + + server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) + self.server = yield server_endpoint.listen(self.site) + self.port_number = self.server.getHost().port + + # Connect H2 client with server + self.client_certificate = get_client_certificate(self.key_file, self.certificate_file) + client_options = optionsForClientTLS( + hostname=self.hostname, + trustRoot=self.client_certificate, + acceptableProtocols=[b'h2'] + ) + uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) + + self.conn_closed_deferred = Deferred() + from scrapy.core.http2.protocol import H2ClientFactory + h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) + client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) + self.client = yield client_endpoint.connect(h2_client_factory) + + @inlineCallbacks + def tearDown(self): + if self.client.connected: + yield self.client.transport.loseConnection() + yield self.client.transport.abortConnection() + yield self.server.stopListening() + shutil.rmtree(self.temp_directory) + self.conn_closed_deferred = None + + def get_url(self, path): + """ + :param path: Should have / at the starting compulsorily if not empty + :return: Complete url + """ + assert len(path) > 0 and (path[0] == '/' or path[0] == '&') + return f'{self.scheme}://{self.hostname}:{self.port_number}{path}' + + def make_request(self, request: Request) -> Deferred: + return self.client.request(request, DummySpider()) + + @staticmethod + def _check_repeat(get_deferred, count): + d_list = [] + for _ in range(count): + d = get_deferred() + d_list.append(d) + + return DeferredList(d_list, fireOnOneErrback=True) + + def _check_GET( + self, + request: Request, + expected_body, + expected_status + ): + def check_response(response: Response): + self.assertEqual(response.status, expected_status) + self.assertEqual(response.body, expected_body) + self.assertEqual(response.request, request) + + content_length = int(response.headers.get('Content-Length')) + self.assertEqual(len(response.body), content_length) + + d = self.make_request(request) + d.addCallback(check_response) + d.addErrback(self.fail) + return d + + def test_GET_small_body(self): + request = Request(self.get_url('/get-data-html-small')) + return self._check_GET(request, Data.HTML_SMALL, 200) + + def test_GET_large_body(self): + request = Request(self.get_url('/get-data-html-large')) + return self._check_GET(request, Data.HTML_LARGE, 200) + + def _check_GET_x10(self, *args, **kwargs): + def get_deferred(): + return self._check_GET(*args, **kwargs) + + return self._check_repeat(get_deferred, 10) + + def test_GET_small_body_x10(self): + return self._check_GET_x10( + Request(self.get_url('/get-data-html-small')), + Data.HTML_SMALL, + 200 + ) + + def test_GET_large_body_x10(self): + return self._check_GET_x10( + Request(self.get_url('/get-data-html-large')), + Data.HTML_LARGE, + 200 + ) + + def _check_POST_json( + self, + request: Request, + expected_request_body, + expected_extra_data, + expected_status: int + ): + d = self.make_request(request) + + def assert_response(response: Response): + self.assertEqual(response.status, expected_status) + self.assertEqual(response.request, request) + + content_length = int(response.headers.get('Content-Length')) + self.assertEqual(len(response.body), content_length) + + # Parse the body + content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + body = json.loads(str(response.body, content_encoding)) + self.assertIn('request-body', body) + self.assertIn('extra-data', body) + self.assertIn('request-headers', body) + + request_body = body['request-body'] + self.assertEqual(request_body, expected_request_body) + + extra_data = body['extra-data'] + self.assertEqual(extra_data, expected_extra_data) + + # Check if headers were sent successfully + request_headers = body['request-headers'] + for k, v in request.headers.items(): + k_str = str(k, 'utf-8') + self.assertIn(k_str, request_headers) + self.assertEqual(request_headers[k_str], str(v[0], 'utf-8')) + + d.addCallback(assert_response) + d.addErrback(self.fail) + return d + + def test_POST_small_json(self): + request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) + return self._check_POST_json( + request, + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + + def test_POST_large_json(self): + request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) + return self._check_POST_json( + request, + Data.JSON_LARGE, + Data.EXTRA_LARGE, + 200 + ) + + def _check_POST_json_x10(self, *args, **kwargs): + def get_deferred(): + return self._check_POST_json(*args, **kwargs) + + return self._check_repeat(get_deferred, 10) + + def test_POST_small_json_x10(self): + request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) + return self._check_POST_json_x10( + request, + Data.JSON_SMALL, + Data.EXTRA_SMALL, + 200 + ) + + def test_POST_large_json_x10(self): + request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) + return self._check_POST_json_x10( + request, + Data.JSON_LARGE, + Data.EXTRA_LARGE, + 200 + ) + + @inlineCallbacks + def test_invalid_negotiated_protocol(self): + with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"): + request = Request(url=self.get_url('/status?n=200')) + with self.assertRaises(ResponseFailed): + yield self.make_request(request) + + def test_cancel_request(self): + request = Request(url=self.get_url('/get-data-html-large')) + + def assert_response(response: Response): + self.assertEqual(response.status, 499) + self.assertEqual(response.request, request) + + d = self.make_request(request) + d.addCallback(assert_response) + d.addErrback(self.fail) + d.cancel() + + return d + + def test_download_maxsize_exceeded(self): + request = Request(url=self.get_url('/get-data-html-large'), meta={'download_maxsize': 1000}) + + def assert_cancelled_error(failure): + self.assertIsInstance(failure.value, CancelledError) + error_pattern = re.compile( + rf'Cancelling download of {request.url}: received response ' + rf'size \(\d*\) larger than download max size \(1000\)' + ) + self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1) + + d = self.make_request(request) + d.addCallback(self.fail) + d.addErrback(assert_cancelled_error) + return d + + def test_received_dataloss_response(self): + """In case when value of Header Content-Length != len(Received Data) + ProtocolError is raised""" + request = Request(url=self.get_url('/dataloss')) + + def assert_failure(failure: Failure): + self.assertTrue(len(failure.value.reasons) > 0) + from h2.exceptions import InvalidBodyLengthError + self.assertTrue(any( + isinstance(error, InvalidBodyLengthError) + for error in failure.value.reasons + )) + + d = self.make_request(request) + d.addCallback(self.fail) + d.addErrback(assert_failure) + return d + + def test_missing_content_length_header(self): + request = Request(url=self.get_url('/no-content-length-header')) + + def assert_content_length(response: Response): + self.assertEqual(response.status, 200) + self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) + self.assertEqual(response.request, request) + self.assertNotIn('Content-Length', response.headers) + + d = self.make_request(request) + d.addCallback(assert_content_length) + d.addErrback(self.fail) + return d + + @inlineCallbacks + def _check_log_warnsize( + self, + request, + warn_pattern, + expected_body + ): + with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm: + response = yield self.make_request(request) + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + self.assertEqual(response.body, expected_body) + + # Check the warning is raised only once for this request + self.assertEqual(sum( + len(re.findall(warn_pattern, log)) + for log in cm.output + ), 1) + + @inlineCallbacks + def test_log_expected_warnsize(self): + request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000}) + warn_pattern = re.compile( + rf'Expected response size \(\d*\) larger than ' + rf'download warn size \(1000\) in request {request}' + ) + + yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) + + @inlineCallbacks + def test_log_received_warnsize(self): + request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10}) + warn_pattern = re.compile( + rf'Received more \(\d*\) bytes than download ' + rf'warn size \(10\) in request {request}' + ) + + yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) + + def test_max_concurrent_streams(self): + """Send 500 requests at one to check if we can handle + very large number of request. + """ + + def get_deferred(): + return self._check_GET( + Request(self.get_url('/get-data-html-small')), + Data.HTML_SMALL, + 200 + ) + + return self._check_repeat(get_deferred, 500) + + def test_inactive_stream(self): + """Here we send 110 requests considering the MAX_CONCURRENT_STREAMS + by default is 100. After sending the first 100 requests we close the + connection.""" + d_list = [] + + def assert_inactive_stream(failure): + self.assertIsNotNone(failure.check(ResponseFailed)) + from scrapy.core.http2.stream import InactiveStreamClosed + self.assertTrue(any( + isinstance(e, InactiveStreamClosed) + for e in failure.value.reasons + )) + + # Send 100 request (we do not check the result) + for _ in range(100): + d = self.make_request(Request(self.get_url('/get-data-html-small'))) + d.addBoth(lambda _: None) + d_list.append(d) + + # Now send 10 extra request and save the response deferred in a list + for _ in range(10): + d = self.make_request(Request(self.get_url('/get-data-html-small'))) + d.addCallback(self.fail) + d.addErrback(assert_inactive_stream) + d_list.append(d) + + # Close the connection now to fire all the extra 10 requests errback + # with InactiveStreamClosed + self.client.transport.loseConnection() + + return DeferredList(d_list, consumeErrors=True, fireOnOneErrback=True) + + def test_invalid_request_type(self): + with self.assertRaises(TypeError): + self.make_request('https://InvalidDataTypePassed.com') + + def test_query_parameters(self): + params = { + 'a': generate_random_string(20), + 'b': generate_random_string(20), + 'c': generate_random_string(20), + 'd': generate_random_string(20) + } + request = Request(self.get_url(f'/query-params?{urlencode(params)}')) + + def assert_query_params(response: Response): + content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + data = json.loads(str(response.body, content_encoding)) + self.assertEqual(data, params) + + d = self.make_request(request) + d.addCallback(assert_query_params) + d.addErrback(self.fail) + + return d + + def test_status_codes(self): + def assert_response_status(response: Response, expected_status: int): + self.assertEqual(response.status, expected_status) + + d_list = [] + for status in [200, 404]: + request = Request(self.get_url(f'/status?n={status}')) + d = self.make_request(request) + d.addCallback(assert_response_status, status) + d.addErrback(self.fail) + d_list.append(d) + + return DeferredList(d_list, fireOnOneErrback=True) + + def test_response_has_correct_certificate_ip_address(self): + request = Request(self.get_url('/status?n=200')) + + def assert_metadata(response: Response): + self.assertEqual(response.request, request) + self.assertIsInstance(response.certificate, Certificate) + self.assertIsNotNone(response.certificate.original) + self.assertEqual(response.certificate.getIssuer(), self.client_certificate.getIssuer()) + self.assertTrue(response.certificate.getPublicKey().matches(self.client_certificate.getPublicKey())) + + self.assertIsInstance(response.ip_address, IPv4Address) + self.assertEqual(str(response.ip_address), '127.0.0.1') + + d = self.make_request(request) + d.addCallback(assert_metadata) + d.addErrback(self.fail) + + return d + + def _check_invalid_netloc(self, url): + request = Request(url) + + def assert_invalid_hostname(failure: Failure): + from scrapy.core.http2.stream import InvalidHostname + self.assertIsNotNone(failure.check(InvalidHostname)) + error_msg = str(failure.value) + self.assertIn('localhost', error_msg) + self.assertIn('127.0.0.1', error_msg) + self.assertIn(str(request), error_msg) + + d = self.make_request(request) + d.addCallback(self.fail) + d.addErrback(assert_invalid_hostname) + return d + + def test_invalid_hostname(self): + return self._check_invalid_netloc('https://notlocalhost.notlocalhostdomain') + + def test_invalid_host_port(self): + port = self.port_number + 1 + return self._check_invalid_netloc(f'https://127.0.0.1:{port}') + + def test_connection_stays_with_invalid_requests(self): + d_list = [ + self.test_invalid_hostname(), + self.test_invalid_host_port(), + self.test_GET_small_body(), + self.test_POST_small_json() + ] + + return DeferredList(d_list, fireOnOneErrback=True) + + def test_connection_timeout(self): + request = Request(self.get_url('/timeout')) + d = self.make_request(request) + + # Update the timer to 1s to test connection timeout + self.client.setTimeout(1) + + def assert_timeout_error(failure: Failure): + for err in failure.value.reasons: + from scrapy.core.http2.protocol import H2ClientProtocol + if isinstance(err, TimeoutError): + self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err)) + break + else: + self.fail() + + d.addCallback(self.fail) + d.addErrback(assert_timeout_error) + return d + + def test_request_headers_received(self): + request = Request(self.get_url('/request-headers'), headers={ + 'header-1': 'header value 1', + 'header-2': 'header value 2' + }) + d = self.make_request(request) + + def assert_request_headers(response: Response): + self.assertEqual(response.status, 200) + self.assertEqual(response.request, request) + + response_headers = json.loads(str(response.body, 'utf-8')) + self.assertIsInstance(response_headers, dict) + for k, v in request.headers.items(): + k, v = str(k, 'utf-8'), str(v[0], 'utf-8') + self.assertIn(k, response_headers) + self.assertEqual(v, response_headers[k]) + + d.addErrback(self.fail) + d.addCallback(assert_request_headers) + return d diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 540e27907..08420332c 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -34,7 +34,6 @@ class WrappedRequestTest(TestCase): self.assertTrue(self.wrapped.unverifiable) def test_get_origin_req_host(self): - self.assertEqual(self.wrapped.get_origin_req_host(), 'www.example.com') self.assertEqual(self.wrapped.origin_req_host, 'www.example.com') def test_has_header(self): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 74579dfc4..579ef9fa2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -379,6 +379,20 @@ class FormRequestTest(RequestTest): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, b'') + def test_formdata_overrides_querystring(self): + data = (('a', 'one'), ('a', 'two'), ('b', '2')) + url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', + method='GET', formdata=data).url.split('#')[0] + fs = _qs(self.request_class(url, method='GET', formdata=data)) + self.assertEqual(set(fs[b'a']), {b'one', b'two'}) + self.assertEqual(fs[b'b'], [b'2']) + self.assertIsNone(fs.get(b'c')) + + data = {'a': '1', 'b': '2'} + fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) + self.assertEqual(fs[b'a'], [b'1']) + self.assertEqual(fs[b'b'], [b'2']) + def test_default_encoding_bytes(self): # using default encoding (utf-8) data = {b'one': b'two', b'price': b'\xc2\xa3 100'} @@ -1203,18 +1217,17 @@ class FormRequestTest(RequestTest): response, formcss="input[name='abc']") def test_from_response_valid_form_methods(self): - body = """ - - """ + form_methods = [[method, method] for method in self.request_class.valid_form_methods] + form_methods.append(['UNKNOWN', 'GET']) - for method in self.request_class.valid_form_methods: - response = _buildresponse(body % method) + for method, expected in form_methods: + response = _buildresponse( + f'
' + '' + '
' + ) r = self.request_class.from_response(response) - self.assertEqual(r.method, method) - - response = _buildresponse(body % 'UNKNOWN') - r = self.request_class.from_response(response) - self.assertEqual(r.method, 'GET') + self.assertEqual(r.method, expected) def _buildresponse(body, **kwargs): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index f831ef5dc..2986f884f 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,10 +1,8 @@ import unittest from unittest import mock -from warnings import catch_warnings from w3lib.encoding import resolve_encoding -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector @@ -19,7 +17,7 @@ class BaseResponseTest(unittest.TestCase): response_class = Response def test_init(self): - # Response requires url in the consturctor + # Response requires url in the constructor self.assertRaises(Exception, self.response_class) self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class)) self.assertRaises(TypeError, self.response_class, b"http://example.com") @@ -134,7 +132,6 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) - self.assertEqual(response.body_as_unicode(), body_unicode) self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): @@ -344,10 +341,6 @@ class TextResponseTest(BaseResponseTest): original_string = unicode_string.encode('cp1251') r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') - # check body_as_unicode - self.assertTrue(isinstance(r1.body_as_unicode(), str)) - self.assertEqual(r1.body_as_unicode(), unicode_string) - # check response.text self.assertTrue(isinstance(r1.text, str)) self.assertEqual(r1.text, unicode_string) @@ -388,7 +381,7 @@ class TextResponseTest(BaseResponseTest): def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" r = self.response_class("http://www.example.com", - headers={"Content-type": ["text/html; charset=UKNOWN"]}, + headers={"Content-type": ["text/html; charset=UNKNOWN"]}, body=b"\xc2\xa3") self.assertEqual(r._declared_encoding(), None) self._assert_response_values(r, 'utf-8', "\xa3") @@ -679,13 +672,6 @@ class TextResponseTest(BaseResponseTest): with self.assertRaises(ValueError): response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]') - def test_body_as_unicode_deprecation_warning(self): - with catch_warnings(record=True) as warnings: - r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8') - self.assertEqual(r1.body_as_unicode(), 'Hello') - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - def test_json_response(self): json_body = b"""{"ip": "109.187.217.200"}""" json_response = self.response_class("http://www.example.com", body=json_body) @@ -816,3 +802,62 @@ class XmlResponseTest(TextResponseTest): response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(), response.selector.xpath("//s2:elem/text()").getall(), ) + + +class CustomResponse(TextResponse): + attributes = TextResponse.attributes + ("foo", "bar") + + def __init__(self, *args, **kwargs) -> None: + self.foo = kwargs.pop("foo", None) + self.bar = kwargs.pop("bar", None) + self.lost = kwargs.pop("lost", None) + super().__init__(*args, **kwargs) + + +class CustomResponseTest(TextResponseTest): + response_class = CustomResponse + + def test_copy(self): + super().test_copy() + r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost") + r2 = r1.copy() + self.assertIsInstance(r2, self.response_class) + self.assertEqual(r1.foo, r2.foo) + self.assertEqual(r1.bar, r2.bar) + self.assertEqual(r1.lost, "lost") + self.assertIsNone(r2.lost) + + def test_replace(self): + super().test_replace() + r1 = self.response_class(url="https://example.org", status=200, foo="foo", bar="bar", lost="lost") + + r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") + self.assertIsInstance(r2, self.response_class) + self.assertEqual(r1.foo, "foo") + self.assertEqual(r1.bar, "bar") + self.assertEqual(r1.lost, "lost") + self.assertEqual(r2.foo, "new-foo") + self.assertEqual(r2.bar, "new-bar") + self.assertEqual(r2.lost, "new-lost") + + r3 = r1.replace(foo="new-foo", bar="new-bar") + self.assertIsInstance(r3, self.response_class) + self.assertEqual(r1.foo, "foo") + self.assertEqual(r1.bar, "bar") + self.assertEqual(r1.lost, "lost") + self.assertEqual(r3.foo, "new-foo") + self.assertEqual(r3.bar, "new-bar") + self.assertIsNone(r3.lost) + + r4 = r1.replace(foo="new-foo") + self.assertIsInstance(r4, self.response_class) + self.assertEqual(r1.foo, "foo") + self.assertEqual(r1.bar, "bar") + self.assertEqual(r1.lost, "lost") + self.assertEqual(r4.foo, "new-foo") + self.assertEqual(r4.bar, "bar") + self.assertIsNone(r4.lost) + + with self.assertRaises(TypeError) as ctx: + r1.replace(unknown="unknown") + self.assertTrue(str(ctx.exception).endswith("__init__() got an unexpected keyword argument 'unknown'")) diff --git a/tests/test_item.py b/tests/test_item.py index 78d204e34..25f2aea0a 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,9 +1,7 @@ import unittest from unittest import mock -from warnings import catch_warnings -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta +from scrapy.item import ABCMeta, Field, Item, ItemMeta class ItemTest(unittest.TestCase): @@ -254,18 +252,6 @@ class ItemTest(unittest.TestCase): item['tags'].append('tag2') assert item['tags'] != copied_item['tags'] - def test_dictitem_deprecation_warning(self): - """Make sure the DictItem deprecation warning is not issued for - Item""" - with catch_warnings(record=True) as warnings: - Item() - self.assertEqual(len(warnings), 0) - - class SubclassedItem(Item): - pass - SubclassedItem() - self.assertEqual(len(warnings), 0) - class ItemMetaTest(unittest.TestCase): @@ -303,92 +289,5 @@ class ItemMetaClassCellRegression(unittest.TestCase): super().__init__(*args, **kwargs) -class DictItemTest(unittest.TestCase): - - def test_deprecation_warning(self): - with catch_warnings(record=True) as warnings: - DictItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: - class SubclassedDictItem(DictItem): - pass - SubclassedDictItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - -class BaseItemTest(unittest.TestCase): - - def test_isinstance_check(self): - - class SubclassedBaseItem(BaseItem): - pass - - class SubclassedItem(Item): - pass - - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) - self.assertTrue(isinstance(Item(), BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), BaseItem)) - - # make sure internal checks using private _BaseItem class succeed - self.assertTrue(isinstance(BaseItem(), _BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), _BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) - - def test_deprecation_warning(self): - """ - Make sure deprecation warnings are logged whenever BaseItem is used, - either instantiated or in an isinstance check - """ - with catch_warnings(record=True) as warnings: - BaseItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - - class SubclassedBaseItem(BaseItem): - pass - - SubclassedBaseItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - self.assertFalse(isinstance("foo", BaseItem)) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - -class ItemNoDeprecationWarningTest(unittest.TestCase): - def test_no_deprecation_warning(self): - """ - Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used. - """ - class SubclassedItem(Item): - pass - - with catch_warnings(record=True) as warnings: - Item() - SubclassedItem() - _BaseItem() - self.assertFalse(isinstance("foo", _BaseItem)) - self.assertFalse(isinstance("foo", Item)) - self.assertFalse(isinstance("foo", SubclassedItem)) - self.assertTrue(isinstance(_BaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), Item)) - self.assertTrue(isinstance(SubclassedItem(), SubclassedItem)) - self.assertEqual(len(warnings), 0) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_loader.py b/tests/test_loader.py index b0bc82f4e..f7ab1f236 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -4,7 +4,7 @@ import attr from itemadapter import ItemAdapter from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst -from scrapy.http import HtmlResponse +from scrapy.http import HtmlResponse, Response from scrapy.item import Item, Field from scrapy.loader import ItemLoader from scrapy.selector import Selector @@ -304,6 +304,12 @@ class SelectortemLoaderTest(unittest.TestCase): l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), ['Marta']) + + def test_init_method_with_base_response(self): + """Selector should be None after initialization""" + response = Response("https://scrapy.org") + l = TestItemLoader(response=response) + self.assertIs(l.selector, None) def test_init_method_with_response(self): l = TestItemLoader(response=self.response) diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 41afa2896..0fd52da5f 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -703,7 +703,7 @@ class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): return None with warnings.catch_warnings(record=True) as w: - wrap_loader_context(function, context=dict()) + wrap_loader_context(function, context={}) assert len(w) == 1 assert issubclass(w[0].category, ScrapyDeprecationWarning) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 55fcfa7ba..f49fda701 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -180,7 +180,18 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) +try: + from PIL import Image # noqa: imported just to check for the import error +except ImportError: + skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' +else: + skip_pillow = None + + class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): + + skip = skip_pillow + pipeline_class = 'scrapy.pipelines.images.ImagesPipeline' store_setting_key = 'IMAGES_STORE' media_key = 'images' diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4e1b90787..4228173ed 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -25,6 +25,7 @@ from scrapy.pipelines.files import ( from scrapy.settings import Settings from scrapy.utils.test import ( assert_gcs_environ, + get_crawler, get_ftp_content_and_delete, get_gcs_content_and_delete, skip_if_no_boto, @@ -47,7 +48,9 @@ class FilesPipelineTestCase(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() - self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})) + settings_dict = {'FILES_STORE': self.tempdir} + crawler = get_crawler(spidercls=None, settings_dict=settings_dict) + self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline.download_func = _mocked_download_func self.pipeline.open_spider(None) @@ -525,6 +528,29 @@ class TestGCSFilesStore(unittest.TestCase): self.assertEqual(blob.content_type, 'application/octet-stream') self.assertIn(expected_policy, acl) + @defer.inlineCallbacks + def test_blob_path_consistency(self): + """Test to make sure that paths used to store files is the same as the one used to get + already uploaded files. + """ + assert_gcs_environ() + try: + import google.cloud.storage # noqa + except ModuleNotFoundError: + raise unittest.SkipTest("google-cloud-storage is not installed") + else: + with mock.patch('google.cloud.storage') as _: + with mock.patch('scrapy.pipelines.files.time') as _: + uri = 'gs://my_bucket/my_prefix/' + store = GCSFilesStore(uri) + store.bucket = mock.Mock() + path = 'full/my_data.txt' + yield store.persist_file(path, mock.Mock(), info=None, meta=None, headers=None) + yield store.stat_file(path, info=None) + expected_blob_path = store.prefix + path + store.bucket.blob.assert_called_with(expected_blob_path) + store.bucket.get_blob.assert_called_with(expected_blob_path) + class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index ad138a2dc..dd94d296b 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -23,15 +23,16 @@ except ImportError: dataclass_field = None -skip = False try: from PIL import Image except ImportError: - skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' + skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' else: encoders = {'jpeg_encoder', 'jpeg_decoder'} if not encoders.issubset(set(Image.core.__dict__)): - skip = 'Missing JPEG encoders' + skip_pillow = 'Missing JPEG encoders' + else: + skip_pillow = None def _mocked_download_func(request, info): @@ -41,7 +42,7 @@ def _mocked_download_func(request, info): class ImagesPipelineTestCase(unittest.TestCase): - skip = skip + skip = skip_pillow def setUp(self): self.tempdir = mkdtemp() @@ -92,6 +93,22 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') + def test_thumbnail_name_from_item(self): + """ + Custom thumbnail name based on item data, overriding default implementation + """ + + class CustomImagesPipeline(ImagesPipeline): + def thumb_path(self, request, thumb_id, response=None, info=None, item=None): + return f"thumb/{thumb_id}/{item.get('path')}" + + thumb_path = CustomImagesPipeline.from_settings(Settings( + {'IMAGES_STORE': self.tempdir} + )).thumb_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file') + def test_convert_image(self): SIZE = (100, 100) # straigh forward case: RGB and JPEG @@ -137,6 +154,8 @@ class DeprecatedImagesPipeline(ImagesPipeline): class ImagesPipelineTestCaseFieldsMixin: + skip = skip_pillow + def test_item_fields_default(self): url = 'http://www.example.com/images/1.jpg' item = self.item_class(name='item1', image_urls=[url]) @@ -221,6 +240,9 @@ class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, u class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): + + skip = skip_pillow + img_cls_attribute_names = [ # Pipeline attribute names with corresponding setting names. ("EXPIRES", "IMAGES_EXPIRES"), diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 6afd47497..84e867660 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,20 +1,31 @@ +from typing import Optional +import io + from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks +from scrapy import signals from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.request import request_fingerprint +from scrapy.pipelines.files import FileException from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline -from scrapy.pipelines.files import FileException +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all -from scrapy import signals +from scrapy.utils.test import get_crawler + + +try: + from PIL import Image # noqa: imported just to check for the import error +except ImportError: + skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' +else: + skip_pillow = None def _mocked_download_func(request, info): @@ -28,11 +39,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase): settings = None def setUp(self): - self.spider = Spider('media.com') - self.pipe = self.pipeline_class(download_func=_mocked_download_func, - settings=Settings(self.settings)) + spider_cls = Spider + self.spider = spider_cls('media.com') + crawler = get_crawler(spider_cls, self.settings) + self.pipe = self.pipeline_class.from_crawler(crawler) + self.pipe.download_func = _mocked_download_func self.pipe.open_spider(self.spider) self.info = self.pipe.spiderinfo + self.fingerprint = crawler.request_fingerprinter.fingerprint def tearDown(self): for name, signal in vars(signals).items(): @@ -145,7 +159,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): self.assertEqual(failure.value.__context__, def_gen_return_exc) # Let's calculate the request fingerprint and fake some runtime data... - fp = request_fingerprint(request) + fp = self.fingerprint(request) info = self.pipe.spiderinfo info.downloading.add(fp) info.waiting[fp] = [] @@ -262,7 +276,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req) # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item - assert request_fingerprint(req) in self.info.downloaded + self.assertIn(self.fingerprint(req), self.info.downloaded) # returns iterable of Requests req1 = Request('http://url1') @@ -270,8 +284,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=iter([req1, req2])) new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item - assert request_fingerprint(req1) in self.info.downloaded - assert request_fingerprint(req2) in self.info.downloaded + assert self.fingerprint(req1) in self.info.downloaded + assert self.fingerprint(req2) in self.info.downloaded @inlineCallbacks def test_results_are_cached_across_multiple_items(self): @@ -287,7 +301,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req2) new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) - self.assertEqual(request_fingerprint(req1), request_fingerprint(req2)) + self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) self.assertEqual(new_item['results'], [(True, rsp1)]) @inlineCallbacks @@ -303,7 +317,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_wait_if_request_is_downloading(self): def _check_downloading(response): - fp = request_fingerprint(req1) + fp = self.fingerprint(req1) self.assertTrue(fp in self.info.downloading) self.assertTrue(fp in self.info.waiting) self.assertTrue(fp not in self.info.downloaded) @@ -340,14 +354,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): def __init__(self, *args, **kwargs): - super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._mockcalled = [] def get_media_requests(self, item, info): item_url = item['image_urls'][0] + output_img = io.BytesIO() + img = Image.new('RGB', (60, 30), color='red') + img.save(output_img, format='JPEG') return Request( item_url, - meta={'response': Response(item_url, status=200, body=b'data')} + meta={'response': Response(item_url, status=200, body=output_img.getvalue())} ) def inc_stats(self, *args, **kwargs): @@ -355,33 +372,44 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): def media_to_download(self, request, info): self._mockcalled.append('media_to_download') - return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info) + return super().media_to_download(request, info) def media_downloaded(self, response, request, info): self._mockcalled.append('media_downloaded') - return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info) + return super().media_downloaded(response, request, info) def file_downloaded(self, response, request, info): self._mockcalled.append('file_downloaded') - return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info) + return super().file_downloaded(response, request, info) def file_path(self, request, response=None, info=None): self._mockcalled.append('file_path') - return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info) + return super().file_path(request, response, info) + + def thumb_path(self, request, thumb_id, response=None, info=None): + self._mockcalled.append('thumb_path') + return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info) def get_images(self, response, request, info): self._mockcalled.append('get_images') - return [] + return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info) def image_downloaded(self, response, request, info): self._mockcalled.append('image_downloaded') - return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info) + return super().image_downloaded(response, request, info) class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): + skip = skip_pillow def setUp(self): - self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) + settings_dict = { + 'IMAGES_STORE': 'store-uri', + 'IMAGES_THUMBS': {'small': (50, 50)}, + } + crawler = get_crawler(spidercls=None, settings_dict=settings_dict) + self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler) + self.pipe.download_func = _mocked_download_func self.pipe.open_spider(None) self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[]) @@ -433,6 +461,16 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): ) self._assert_method_called_with_warnings('file_path', message, warnings) + @inlineCallbacks + def test_thumb_path_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, ' + 'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)' + ) + self._assert_method_called_with_warnings('thumb_path', message, warnings) + @inlineCallbacks def test_get_images_called(self): yield self.pipe.process_item(self.item, None) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index ff3af9a74..8e432b913 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -6,6 +6,7 @@ from twisted.internet.defer import Deferred from twisted.trial import unittest from scrapy import Spider, signals, Request +from scrapy.utils.defer import maybe_deferred_to_future, deferred_to_future from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver import MockServer @@ -31,18 +32,38 @@ class DeferredPipeline: class AsyncDefPipeline: async def process_item(self, item, spider): - await defer.succeed(42) + d = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, None) + await maybe_deferred_to_future(d) item['pipeline_passed'] = True return item class AsyncDefAsyncioPipeline: async def process_item(self, item, spider): + d = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, None) + await deferred_to_future(d) await asyncio.sleep(0.2) item['pipeline_passed'] = await get_from_asyncio_queue(True) return item +class AsyncDefNotAsyncioPipeline: + async def process_item(self, item, spider): + d1 = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d1.callback, None) + await d1 + d2 = Deferred() + reactor.callLater(0, d2.callback, None) + await maybe_deferred_to_future(d2) + item['pipeline_passed'] = True + return item + + class ItemSpider(Spider): name = 'itemspider' @@ -99,3 +120,10 @@ class PipelineTestCase(unittest.TestCase): crawler = self._create_crawler(AsyncDefAsyncioPipeline) yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 1) + + @mark.only_not_asyncio() + @defer.inlineCallbacks + def test_asyncdef_not_asyncio_pipeline(self): + crawler = self._create_crawler(AsyncDefNotAsyncioPipeline) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(len(self.items), 1) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py new file mode 100644 index 000000000..ec55033d1 --- /dev/null +++ b/tests/test_pqueues.py @@ -0,0 +1,144 @@ +import tempfile +import unittest + +import queuelib + +from scrapy.http.request import Request +from scrapy.pqueues import ScrapyPriorityQueue, DownloaderAwarePriorityQueue +from scrapy.spiders import Spider +from scrapy.squeues import FifoMemoryQueue +from scrapy.utils.test import get_crawler + +from tests.test_scheduler import MockDownloader, MockEngine + + +class PriorityQueueTest(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("foo") + + def test_queue_push_pop_one(self): + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + self.assertIsNone(queue.pop()) + self.assertEqual(len(queue), 0) + req1 = Request("https://example.org/1", priority=1) + queue.push(req1) + self.assertEqual(len(queue), 1) + dequeued = queue.pop() + self.assertEqual(len(queue), 0) + self.assertEqual(dequeued.url, req1.url) + self.assertEqual(dequeued.priority, req1.priority) + self.assertEqual(queue.close(), []) + + def test_no_peek_raises(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + queue.push(Request("https://example.org")) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + queue.peek() + queue.close() + + def test_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir) + self.assertEqual(len(queue), 0) + self.assertIsNone(queue.peek()) + req1 = Request("https://example.org/1") + req2 = Request("https://example.org/2") + req3 = Request("https://example.org/3") + queue.push(req1) + queue.push(req2) + queue.push(req3) + self.assertEqual(len(queue), 3) + self.assertEqual(queue.peek().url, req1.url) + self.assertEqual(queue.pop().url, req1.url) + self.assertEqual(len(queue), 2) + self.assertEqual(queue.peek().url, req2.url) + self.assertEqual(queue.pop().url, req2.url) + self.assertEqual(len(queue), 1) + self.assertEqual(queue.peek().url, req3.url) + self.assertEqual(queue.pop().url, req3.url) + self.assertEqual(queue.close(), []) + + def test_queue_push_pop_priorities(self): + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]) + self.assertIsNone(queue.pop()) + self.assertEqual(len(queue), 0) + req1 = Request("https://example.org/1", priority=1) + req2 = Request("https://example.org/2", priority=2) + req3 = Request("https://example.org/3", priority=3) + queue.push(req1) + queue.push(req2) + queue.push(req3) + self.assertEqual(len(queue), 3) + dequeued = queue.pop() + self.assertEqual(len(queue), 2) + self.assertEqual(dequeued.url, req3.url) + self.assertEqual(dequeued.priority, req3.priority) + self.assertEqual(queue.close(), [-1, -2]) + + +class DownloaderAwarePriorityQueueTest(unittest.TestCase): + def setUp(self): + crawler = get_crawler(Spider) + crawler.engine = MockEngine(downloader=MockDownloader()) + self.queue = DownloaderAwarePriorityQueue.from_crawler( + crawler=crawler, + downstream_queue_cls=FifoMemoryQueue, + key="foo/bar", + ) + + def tearDown(self): + self.queue.close() + + def test_push_pop(self): + self.assertEqual(len(self.queue), 0) + self.assertIsNone(self.queue.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + self.queue.push(req1) + self.queue.push(req2) + self.queue.push(req3) + self.assertEqual(len(self.queue), 3) + self.assertEqual(self.queue.pop().url, req1.url) + self.assertEqual(len(self.queue), 2) + self.assertEqual(self.queue.pop().url, req2.url) + self.assertEqual(len(self.queue), 1) + self.assertEqual(self.queue.pop().url, req3.url) + self.assertEqual(len(self.queue), 0) + self.assertIsNone(self.queue.pop()) + + def test_no_peek_raises(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined") + self.queue.push(Request("https://example.org")) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + self.queue.peek() + + def test_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined") + self.assertEqual(len(self.queue), 0) + req1 = Request("https://example.org/1") + req2 = Request("https://example.org/2") + req3 = Request("https://example.org/3") + self.queue.push(req1) + self.queue.push(req2) + self.queue.push(req3) + self.assertEqual(len(self.queue), 3) + self.assertEqual(self.queue.peek().url, req1.url) + self.assertEqual(self.queue.pop().url, req1.url) + self.assertEqual(len(self.queue), 2) + self.assertEqual(self.queue.peek().url, req2.url) + self.assertEqual(self.queue.pop().url, req2.url) + self.assertEqual(len(self.queue), 1) + self.assertEqual(self.queue.peek().url, req3.url) + self.assertEqual(self.queue.pop().url, req3.url) + self.assertIsNone(self.queue.peek()) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 9eabe6b49..afdfb2578 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,12 +1,9 @@ import json import os -import platform import re import sys from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit -from unittest import skipIf - from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -57,13 +54,14 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -@skipIf("pypy" in sys.executable, - "mitmproxy does not support PyPy") -@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), - "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): def setUp(self): + try: + import mitmproxy # noqa: F401 + except ImportError: + self.skipTest('mitmproxy is not installed') + self.mockserver = MockServer() self.mockserver.__enter__() self._oldenv = os.environ.copy() diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 00c532c41..25d9657d5 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -106,9 +106,9 @@ class CrawlTestCase(TestCase): """ Downloader middleware which returns a response with an specific 'request' attribute. - * The spider callback should receive the overriden response.request - * Handlers listening to the response_received signal should receive the overriden response.request - * The "crawled" log message should show the overriden response.request + * The spider callback should receive the overridden response.request + * Handlers listening to the response_received signal should receive the overridden response.request + * The "crawled" log message should show the overridden response.request """ signal_params = {} @@ -144,7 +144,7 @@ class CrawlTestCase(TestCase): An exception is raised but caught by the next middleware, which returns a Response with a specific 'request' attribute. - The spider callback should receive the overriden response.request + The spider callback should receive the overridden response.request """ url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 145a4e9b2..473a93e69 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -57,10 +57,10 @@ class KeywordArgumentsSpider(MockServerSpider): }, } - checks = list() + checks = [] def start_requests(self): - data = {'key': 'value', 'number': 123} + data = {'key': 'value', 'number': 123, 'callback': 'some_callback'} yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data) yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data) yield Request(self.mockserver.url('/general_without'), self.parse_general) @@ -88,7 +88,8 @@ class KeywordArgumentsSpider(MockServerSpider): if response.url.endswith('/general_with'): self.checks.append(kwargs['key'] == 'value') self.checks.append(kwargs['number'] == 123) - self.crawler.stats.inc_value('boolean_checks', 2) + self.checks.append(kwargs['callback'] == 'some_callback') + self.crawler.stats.inc_value('boolean_checks', 3) elif response.url.endswith('/general_without'): self.checks.append(kwargs == {}) self.crawler.stats.inc_value('boolean_checks') @@ -104,13 +105,13 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(default == 99) self.crawler.stats.inc_value('boolean_checks', 4) - def parse_takes_less(self, response, key): + def parse_takes_less(self, response, key, callback): """ Should raise TypeError: parse_takes_less() got an unexpected keyword argument 'number' """ - def parse_takes_more(self, response, key, number, other): + def parse_takes_more(self, response, key, number, callback, other): """ Should raise TypeError: parse_takes_more() missing 1 required positional argument: 'other' @@ -158,12 +159,16 @@ class CallbackKeywordArgumentsTestCase(TestCase): if key in line.getMessage(): exceptions[key] = line self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) - self.assertEqual( - str(exceptions['takes_less'].exc_info[1]), - "parse_takes_less() got an unexpected keyword argument 'number'" + self.assertTrue( + str(exceptions['takes_less'].exc_info[1]).endswith( + "parse_takes_less() got an unexpected keyword argument 'number'" + ), + msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]), ) self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) - self.assertEqual( - str(exceptions['takes_more'].exc_info[1]), - "parse_takes_more() missing 1 required positional argument: 'other'" + self.assertTrue( + str(exceptions['takes_more'].exc_info[1]).endswith( + "parse_takes_more() missing 1 required positional argument: 'other'" + ), + msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]), ) diff --git a/tests/test_utils_reqser.py b/tests/test_request_dict.py similarity index 68% rename from tests/test_utils_reqser.py rename to tests/test_request_dict.py index ee68cf6b1..5bdcb975b 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_request_dict.py @@ -1,8 +1,16 @@ +import sys import unittest +import warnings +from contextlib import suppress -from scrapy.http import Request, FormRequest -from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy import Spider, Request +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.http import FormRequest, JsonRequest +from scrapy.utils.request import request_from_dict + + +class CustomRequest(Request): + pass class RequestSerializationTest(unittest.TestCase): @@ -27,7 +35,8 @@ class RequestSerializationTest(unittest.TestCase): priority=20, meta={'a': 'b'}, cb_kwargs={'k': 'v'}, - flags=['testFlag']) + flags=['testFlag'], + ) self._assert_serializes_ok(r, spider=self.spider) def test_latin1_body(self): @@ -39,7 +48,7 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r) def _assert_serializes_ok(self, request, spider=None): - d = request_to_dict(request, spider=spider) + d = request.to_dict(spider=spider) request2 = request_from_dict(d, spider=spider) self._assert_same_request(request, request2) @@ -54,16 +63,21 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.cookies, r2.cookies) self.assertEqual(r1.meta, r2.meta) self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) + self.assertEqual(r1.encoding, r2.encoding) self.assertEqual(r1._encoding, r2._encoding) self.assertEqual(r1.priority, r2.priority) self.assertEqual(r1.dont_filter, r2.dont_filter) self.assertEqual(r1.flags, r2.flags) + if isinstance(r1, JsonRequest): + self.assertEqual(r1.dumps_kwargs, r2.dumps_kwargs) def test_request_class(self): - r = FormRequest("http://www.example.com") - self._assert_serializes_ok(r, spider=self.spider) - r = CustomRequest("http://www.example.com") - self._assert_serializes_ok(r, spider=self.spider) + r1 = FormRequest("http://www.example.com") + self._assert_serializes_ok(r1, spider=self.spider) + r2 = CustomRequest("http://www.example.com") + self._assert_serializes_ok(r2, spider=self.spider) + r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4}) + self._assert_serializes_ok(r3, spider=self.spider) def test_callback_serialization(self): r = Request("http://www.example.com", callback=self.spider.parse_item, @@ -75,7 +89,7 @@ class RequestSerializationTest(unittest.TestCase): callback=self.spider.parse_item_reference, errback=self.spider.handle_error_reference) self._assert_serializes_ok(r, spider=self.spider) - request_dict = request_to_dict(r, self.spider) + request_dict = r.to_dict(spider=self.spider) self.assertEqual(request_dict['callback'], 'parse_item_reference') self.assertEqual(request_dict['errback'], 'handle_error_reference') @@ -84,7 +98,7 @@ class RequestSerializationTest(unittest.TestCase): callback=self.spider._TestSpider__parse_item_reference, errback=self.spider._TestSpider__handle_error_reference) self._assert_serializes_ok(r, spider=self.spider) - request_dict = request_to_dict(r, self.spider) + request_dict = r.to_dict(spider=self.spider) self.assertEqual(request_dict['callback'], '_TestSpider__parse_item_reference') self.assertEqual(request_dict['errback'], @@ -110,18 +124,16 @@ class RequestSerializationTest(unittest.TestCase): def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) - self.assertRaises(ValueError, request_to_dict, r) - self.assertRaises(ValueError, request_to_dict, r, spider=self.spider) + self.assertRaises(ValueError, r.to_dict, spider=self.spider) def test_unserializable_callback2(self): r = Request("http://www.example.com", callback=self.spider.parse_item) - self.assertRaises(ValueError, request_to_dict, r) + self.assertRaises(ValueError, r.to_dict, spider=None) def test_unserializable_callback3(self): """Parser method is removed or replaced dynamically.""" class MySpider(Spider): - name = 'my_spider' def parse(self, response): @@ -130,7 +142,35 @@ class RequestSerializationTest(unittest.TestCase): spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) setattr(spider, 'parse', None) - self.assertRaises(ValueError, request_to_dict, r, spider=spider) + self.assertRaises(ValueError, r.to_dict, spider=spider) + + def test_callback_not_available(self): + """Callback method is not available in the spider passed to from_dict""" + spider = TestSpiderDelegation() + r = Request("http://www.example.com", callback=spider.delegated_callback) + d = r.to_dict(spider=spider) + self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) + + +class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): + def _assert_serializes_ok(self, request, spider=None): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with suppress(KeyError): + del sys.modules["scrapy.utils.reqser"] # delete module to reset the deprecation warning + + from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict + + request_copy = _from_dict(_to_dict(request, spider), spider) + self._assert_same_request(request, request_copy) + + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) + self.assertEqual( + "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" + " and/or scrapy.utils.request.request_from_dict instead", + str(caught[0].message), + ) class TestSpiderMixin: @@ -177,7 +217,3 @@ class TestSpider(Spider, TestSpiderMixin): def __parse_item_private(self, response): pass - - -class CustomRequest(Request): - pass diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 373b2e49c..4d4483881 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -22,7 +22,7 @@ class SignalCatcherSpider(Spider): return spider def on_request_left(self, request, spider): - self.caught_times = self.caught_times + 1 + self.caught_times += 1 class TestCatching(TestCase): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 512a7460e..2d4bfa165 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -22,7 +22,7 @@ MockSlot = collections.namedtuple('MockSlot', ['active']) class MockDownloader: def __init__(self): - self.slots = dict() + self.slots = {} def _get_slot_key(self, request, spider): if Downloader.DOWNLOAD_SLOT in request.meta: @@ -31,7 +31,7 @@ class MockDownloader: return urlparse_cached(request).hostname or '' def increment(self, slot_key): - slot = self.slots.setdefault(slot_key, MockSlot(active=list())) + slot = self.slots.setdefault(slot_key, MockSlot(active=[])) slot.active.append(1) def decrement(self, slot_key): @@ -114,7 +114,7 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): for url, priority in _PRIORITIES: self.scheduler.enqueue_request(Request(url, priority=priority)) - priorities = list() + priorities = [] while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) @@ -167,7 +167,7 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): self.close_scheduler() self.create_scheduler() - priorities = list() + priorities = [] while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) @@ -259,7 +259,7 @@ class DownloaderAwareSchedulerTestMixin: self.close_scheduler() self.create_scheduler() - dequeued_slots = list() + dequeued_slots = [] requests = [] downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py new file mode 100644 index 000000000..bf90b4320 --- /dev/null +++ b/tests/test_scheduler_base.py @@ -0,0 +1,159 @@ +from typing import Dict, Optional +from unittest import TestCase +from urllib.parse import urljoin, urlparse + +from testfixtures import LogCapture +from twisted.internet import defer +from twisted.trial.unittest import TestCase as TwistedTestCase + +from scrapy.core.scheduler import BaseScheduler +from scrapy.crawler import CrawlerRunner +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.request import request_fingerprint + +from tests.mockserver import MockServer + + +PATHS = ["/a", "/b", "/c"] +URLS = [urljoin("https://example.org", p) for p in PATHS] + + +class MinimalScheduler: + def __init__(self) -> None: + self.requests: Dict[str, Request] = {} + + def has_pending_requests(self) -> bool: + return bool(self.requests) + + def enqueue_request(self, request: Request) -> bool: + fp = request_fingerprint(request) + if fp not in self.requests: + self.requests[fp] = request + return True + return False + + def next_request(self) -> Optional[Request]: + if self.has_pending_requests(): + fp, request = self.requests.popitem() + return request + return None + + +class SimpleScheduler(MinimalScheduler): + def open(self, spider: Spider) -> defer.Deferred: + return defer.succeed("open") + + def close(self, reason: str) -> defer.Deferred: + return defer.succeed("close") + + def __len__(self) -> int: + return len(self.requests) + + +class TestSpider(Spider): + name = "test" + + def __init__(self, mockserver, *args, **kwargs): + super().__init__(*args, **kwargs) + self.start_urls = map(mockserver.url, PATHS) + + def parse(self, response): + return {"path": urlparse(response.url).path} + + +class InterfaceCheckMixin: + def test_scheduler_class(self): + self.assertTrue(isinstance(self.scheduler, BaseScheduler)) + self.assertTrue(issubclass(self.scheduler.__class__, BaseScheduler)) + + +class BaseSchedulerTest(TestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = BaseScheduler() + + def test_methods(self): + self.assertIsNone(self.scheduler.open(Spider("foo"))) + self.assertIsNone(self.scheduler.close("finished")) + self.assertRaises(NotImplementedError, self.scheduler.has_pending_requests) + self.assertRaises(NotImplementedError, self.scheduler.enqueue_request, Request("https://example.org")) + self.assertRaises(NotImplementedError, self.scheduler.next_request) + + +class MinimalSchedulerTest(TestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = MinimalScheduler() + + def test_open_close(self): + with self.assertRaises(AttributeError): + self.scheduler.open(Spider("foo")) + with self.assertRaises(AttributeError): + self.scheduler.close("finished") + + def test_len(self): + with self.assertRaises(AttributeError): + self.scheduler.__len__() + with self.assertRaises(TypeError): + len(self.scheduler) + + def test_enqueue_dequeue(self): + self.assertFalse(self.scheduler.has_pending_requests()) + for url in URLS: + self.assertTrue(self.scheduler.enqueue_request(Request(url))) + self.assertFalse(self.scheduler.enqueue_request(Request(url))) + self.assertTrue(self.scheduler.has_pending_requests) + + dequeued = [] + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + dequeued.append(request.url) + self.assertEqual(set(dequeued), set(URLS)) + self.assertFalse(self.scheduler.has_pending_requests()) + + +class SimpleSchedulerTest(TwistedTestCase, InterfaceCheckMixin): + def setUp(self): + self.scheduler = SimpleScheduler() + + @defer.inlineCallbacks + def test_enqueue_dequeue(self): + open_result = yield self.scheduler.open(Spider("foo")) + self.assertEqual(open_result, "open") + self.assertFalse(self.scheduler.has_pending_requests()) + + for url in URLS: + self.assertTrue(self.scheduler.enqueue_request(Request(url))) + self.assertFalse(self.scheduler.enqueue_request(Request(url))) + + self.assertTrue(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), len(URLS)) + + dequeued = [] + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + dequeued.append(request.url) + self.assertEqual(set(dequeued), set(URLS)) + + self.assertFalse(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), 0) + + close_result = yield self.scheduler.close("") + self.assertEqual(close_result, "close") + + +class MinimalSchedulerCrawlTest(TwistedTestCase): + scheduler_cls = MinimalScheduler + + @defer.inlineCallbacks + def test_crawl(self): + with MockServer() as mockserver: + settings = {"SCHEDULER": self.scheduler_cls} + with LogCapture() as log: + yield CrawlerRunner(settings).crawl(TestSpider, mockserver) + for path in PATHS: + self.assertIn(f"{{'path': '{path}'}}", str(log)) + self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log)) + + +class SimpleSchedulerCrawlTest(MinimalSchedulerCrawlTest): + scheduler_cls = SimpleScheduler diff --git a/tests/test_spider.py b/tests/test_spider.py index d23543f6a..689349999 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -21,6 +21,7 @@ from scrapy.spiders import ( ) from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler +from tests import get_testdata class SpiderTest(unittest.TestCase): @@ -167,6 +168,23 @@ class CSVFeedSpiderTest(SpiderTest): spider_class = CSVFeedSpider + def test_parse_rows(self): + body = get_testdata('feeds', 'feed-sample6.csv') + response = Response("http://example.org/dummy.csv", body=body) + + class _CrawlSpider(self.spider_class): + name = "test" + delimiter = "," + quotechar = "'" + + def parse_row(self, response, row): + return row + + spider = _CrawlSpider() + rows = list(spider.parse_rows(response)) + assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'} + assert len(rows) == 4 + class CrawlSpiderTest(SpiderTest): @@ -584,39 +602,6 @@ class DeprecationTest(unittest.TestCase): assert issubclass(CrawlSpider, Spider) assert isinstance(CrawlSpider(name='foo'), Spider) - def test_make_requests_from_url_deprecated(self): - class MySpider4(Spider): - name = 'spider1' - start_urls = ['http://example.com'] - - class MySpider5(Spider): - name = 'spider2' - start_urls = ['http://example.com'] - - def make_requests_from_url(self, url): - return Request(url + "/foo", dont_filter=True) - - with warnings.catch_warnings(record=True) as w: - # spider without overridden make_requests_from_url method - # doesn't issue a warning - spider1 = MySpider4() - self.assertEqual(len(list(spider1.start_requests())), 1) - self.assertEqual(len(w), 0) - - # spider without overridden make_requests_from_url method - # should issue a warning when called directly - request = spider1.make_requests_from_url("http://www.example.com") - self.assertTrue(isinstance(request, Request)) - self.assertEqual(len(w), 1) - - # spider with overridden make_requests_from_url issues a warning, - # but the method still works - spider2 = MySpider5() - requests = list(spider2.start_requests()) - self.assertEqual(len(requests), 1) - self.assertEqual(requests[0].url, 'http://example.com/foo') - self.assertEqual(len(w), 2) - class NoParseMethodSpiderTest(unittest.TestCase): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 4929f1e3e..8a35e9fd7 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -118,6 +118,11 @@ class SpiderLoaderTest(unittest.TestCase): settings = Settings({'SPIDER_MODULES': [module], 'SPIDER_LOADER_WARN_ONLY': True}) spider_loader = SpiderLoader.from_settings(settings) + if str(w[0].message).startswith("_SixMetaPathImporter"): + # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, + # at least until all six versions we can import (including botocore.vendored.six) + # are updated to 1.16.0+ + w.pop(0) self.assertIn("Could not load spiders from module", str(w[0].message)) spiders = spider_loader.list() diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 78e926adc..b39576996 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -15,7 +15,7 @@ class SpiderMiddlewareTestCase(TestCase): def setUp(self): self.request = Request('http://example.com/index.html') self.response = Response(self.request.url, request=self.request) - self.crawler = get_crawler(Spider) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}}) self.spider = self.crawler._create_spider('foo') self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e449cd706..46f74ae52 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -139,6 +139,19 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) + def test_httperror_allow_all_false(self): + crawler = get_crawler(_HttpErrorSpider) + mw = HttpErrorMiddleware.from_crawler(crawler) + request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False}) + request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True}) + res404 = self.res404.copy() + res404.request = request_httpstatus_false + res402 = self.res402.copy() + res402.request = request_httpstatus_true + + self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) + self.assertIsNone(mw.process_spider_input(res402, self.spider)) + class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def setUp(self): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 5ef2b23fd..171f4ddfd 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,20 +1,41 @@ from unittest import TestCase +from testfixtures import LogCapture + from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from scrapy.settings import Settings class TestUrlLengthMiddleware(TestCase): - def test_process_spider_output(self): - res = Response('http://scrapytest.org') + def setUp(self): + self.maxlength = 25 + settings = Settings({'URLLENGTH_LIMIT': self.maxlength}) - short_url_req = Request('http://scrapytest.org/') - long_url_req = Request('http://scrapytest.org/this_is_a_long_url') - reqs = [short_url_req, long_url_req] + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') + self.stats = crawler.stats + self.mw = UrlLengthMiddleware.from_settings(settings) - mw = UrlLengthMiddleware(maxlength=25) - spider = Spider('foo') - out = list(mw.process_spider_output(res, reqs, spider)) - self.assertEqual(out, [short_url_req]) + self.response = Response('http://scrapytest.org') + self.short_url_req = Request('http://scrapytest.org/') + self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url') + self.reqs = [self.short_url_req, self.long_url_req] + + def process_spider_output(self): + return list(self.mw.process_spider_output(self.response, self.reqs, self.spider)) + + def test_middleware_works(self): + self.assertEqual(self.process_spider_output(), [self.short_url_req]) + + def test_logging(self): + with LogCapture() as log: + self.process_spider_output() + + ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider) + self.assertEqual(ric, 1) + + self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log)) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index becacce62..acc821b83 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -3,10 +3,10 @@ import sys from queuelib.tests import test_queue as t from scrapy.squeues import ( - MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue, - MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue, - PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue, - PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue + _MarshalFifoSerializationDiskQueue, + _MarshalLifoSerializationDiskQueue, + _PickleFifoSerializationDiskQueue, + _PickleLifoSerializationDiskQueue, ) from scrapy.item import Item, Field from scrapy.http import Request @@ -53,7 +53,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): chunksize = 100000 def queue(self): - return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize) + return _MarshalFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize) class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): @@ -77,7 +77,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): chunksize = 100000 def queue(self): - return PickleFifoDiskQueue(self.qpath, chunksize=self.chunksize) + return _PickleFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize) def test_serialize_item(self): q = self.queue() @@ -155,13 +155,13 @@ class LifoDiskQueueTestMixin: class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def queue(self): - return MarshalLifoDiskQueue(self.qpath) + return _MarshalLifoSerializationDiskQueue(self.qpath) class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def queue(self): - return PickleLifoDiskQueue(self.qpath) + return _PickleLifoSerializationDiskQueue(self.qpath) def test_serialize_item(self): q = self.queue() diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py new file mode 100644 index 000000000..c5fcc1853 --- /dev/null +++ b/tests/test_squeues_request.py @@ -0,0 +1,214 @@ +import shutil +import tempfile +import unittest + +import queuelib + +from scrapy.squeues import ( + PickleFifoDiskQueue, + PickleLifoDiskQueue, + MarshalFifoDiskQueue, + MarshalLifoDiskQueue, + FifoMemoryQueue, + LifoMemoryQueue, +) +from scrapy.http import Request +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler + + +""" +Queues that handle requests +""" + + +class BaseQueueTestCase(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") + self.qpath = self.tempfilename() + self.qdir = self.mkdtemp() + self.crawler = get_crawler(Spider) + + def tearDown(self): + shutil.rmtree(self.tmpdir) + + def tempfilename(self): + with tempfile.NamedTemporaryFile(dir=self.tmpdir) as nf: + return nf.name + + def mkdtemp(self): + return tempfile.mkdtemp(dir=self.tmpdir) + + +class RequestQueueTestMixin: + def queue(self): + raise NotImplementedError() + + def test_one_element_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req = Request("http://www.example.com") + q.push(req) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req.url) + self.assertEqual(q.pop().url, req.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_one_element_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req = Request("http://www.example.com") + q.push(req) + self.assertEqual(len(q), 1) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(q.pop().url, req.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class FifoQueueMixin(RequestQueueTestMixin): + def test_fifo_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + self.assertEqual(len(q), 3) + self.assertEqual(q.peek().url, req1.url) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.peek().url, req2.url) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req3.url) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_fifo_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(len(q), 3) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class LifoQueueMixin(RequestQueueTestMixin): + def test_lifo_with_peek(self): + if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + self.assertEqual(len(q), 3) + self.assertEqual(q.peek().url, req3.url) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.peek().url, req2.url) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.peek().url, req1.url) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.peek()) + self.assertIsNone(q.pop()) + q.close() + + def test_lifo_without_peek(self): + if hasattr(queuelib.queue.FifoMemoryQueue, "peek"): + raise unittest.SkipTest("The queuelib queues do not define peek") + q = self.queue() + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + req1 = Request("http://www.example.com/1") + req2 = Request("http://www.example.com/2") + req3 = Request("http://www.example.com/3") + q.push(req1) + q.push(req2) + q.push(req3) + with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"): + q.peek() + self.assertEqual(len(q), 3) + self.assertEqual(q.pop().url, req3.url) + self.assertEqual(len(q), 2) + self.assertEqual(q.pop().url, req2.url) + self.assertEqual(len(q), 1) + self.assertEqual(q.pop().url, req1.url) + self.assertEqual(len(q), 0) + self.assertIsNone(q.pop()) + q.close() + + +class PickleFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return PickleFifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/fifo") + + +class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return PickleLifoDiskQueue.from_crawler(crawler=self.crawler, key="pickle/lifo") + + +class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return MarshalFifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/fifo") + + +class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return MarshalLifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/lifo") + + +class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase): + def queue(self): + return FifoMemoryQueue.from_crawler(crawler=self.crawler) + + +class LifoMemoryQueueRequestTest(LifoQueueMixin, BaseQueueTestCase): + def queue(self): + return LifoMemoryQueue.from_crawler(crawler=self.crawler) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a2114bd18..295323e4d 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,6 +1,4 @@ -import platform -import sys -from unittest import skipIf, TestCase +from unittest import TestCase from pytest import mark @@ -14,9 +12,6 @@ class AsyncioTest(TestCase): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index dc2560add..a92880626 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -176,7 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": (1, 2, 3, 4), "batch_item_count": 2, - "item_export_kwargs": dict(), + "item_export_kwargs": {}, }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -199,7 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": None, "batch_item_count": 2, - "item_export_kwargs": dict(), + "item_export_kwargs": {}, }) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index e60242a3b..032dbc8c5 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,8 +1,10 @@ +from pytest import mark from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python.failure import Failure from scrapy.utils.defer import ( + deferred_f_from_coro_f, iter_errback, mustbe_deferred, process_chain, @@ -21,7 +23,7 @@ class MustbeDeferredTest(unittest.TestCase): dfd = mustbe_deferred(_append, 1) dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + steps.append(2) # add another value, that should be caught by assertEqual return dfd def test_unfired_deferred(self): @@ -35,7 +37,7 @@ class MustbeDeferredTest(unittest.TestCase): dfd = mustbe_deferred(_append, 1) dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + steps.append(2) # add another value, that should be caught by assertEqual return dfd @@ -117,3 +119,18 @@ class IterErrbackTest(unittest.TestCase): self.assertEqual(out, [0, 1, 2, 3, 4]) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0].value, ZeroDivisionError) + + +class AsyncDefTestsuiteTest(unittest.TestCase): + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f(self): + pass + + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f_generator(self): + yield + + @mark.xfail(reason="Checks that the test is actually executed", strict=True) + @deferred_f_from_coro_f + async def test_deferred_f_from_coro_f_xfail(self): + raise Exception("This is expected to be raised") diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 35d35b45d..e47afa266 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -108,7 +108,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): # ignore subclassing warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', ScrapyDeprecationWarning) + warnings.simplefilter('ignore', MyWarning) class UserClass(Deprecated): pass diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 7148185f4..4943731cb 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -3,10 +3,11 @@ from os.path import join from w3lib.encoding import html_to_unicode -from scrapy.utils.gz import gunzip, is_gzipped -from scrapy.http import Response, Headers +from scrapy.utils.gz import gunzip, gzip_magic_number +from scrapy.http import Response from tests import tests_datadir + SAMPLEDIR = join(tests_datadir, 'compressed') @@ -14,8 +15,12 @@ class GunzipTest(unittest.TestCase): def test_gunzip_basic(self): with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: - text = gunzip(f.read()) - self.assertEqual(len(text), 9950) + r1 = Response("http://www.example.com", body=f.read()) + self.assertTrue(gzip_magic_number(r1)) + + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + self.assertFalse(gzip_magic_number(r2)) + self.assertEqual(len(r2.body), 9950) def test_gunzip_truncated(self): with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f: @@ -28,46 +33,16 @@ class GunzipTest(unittest.TestCase): def test_gunzip_truncated_short(self): with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f: - text = gunzip(f.read()) - assert text.endswith(b'') + r1 = Response("http://www.example.com", body=f.read()) + self.assertTrue(gzip_magic_number(r1)) - def test_is_x_gzipped_right(self): - hdrs = Headers({"Content-Type": "application/x-gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - def test_is_gzipped_right(self): - hdrs = Headers({"Content-Type": "application/gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - def test_is_gzipped_not_quite(self): - hdrs = Headers({"Content-Type": "application/gzippppp"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_case_insensitive(self): - hdrs = Headers({"Content-Type": "Application/X-Gzip"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) - - hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b'') + self.assertFalse(gzip_magic_number(r2)) def test_is_gzipped_empty(self): r1 = Response("http://www.example.com") - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_wrong(self): - hdrs = Headers({"Content-Type": "application/javascript"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertFalse(is_gzipped(r1)) - - def test_is_gzipped_with_charset(self): - hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) - r1 = Response("http://www.example.com", headers=hdrs) - self.assertTrue(is_gzipped(r1)) + self.assertFalse(gzip_magic_number(r1)) def test_gunzip_illegal_eof(self): with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f: diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py deleted file mode 100644 index 363b015a8..000000000 --- a/tests/test_utils_http.py +++ /dev/null @@ -1,19 +0,0 @@ -import unittest - -from scrapy.utils.http import decode_chunked_transfer - - -class ChunkedTest(unittest.TestCase): - - def test_decode_chunked_transfer(self): - """Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding""" - chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n" - chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n" - chunked_body += "3\r\n" + "con\r\n" - chunked_body += "8\r\n" + "sequence\r\n" - chunked_body += "0\r\n\r\n" - body = decode_chunked_transfer(chunked_body) - self.assertEqual( - body, - "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" - ) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index e95a3a316..b83c1d6f0 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -4,7 +4,7 @@ import unittest from unittest import mock from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, rel_has_nofollow, set_environ, walk_modules __doctests__ = ['scrapy.utils.misc'] @@ -27,7 +27,7 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') - self.assertRaises(TypeError, load_object, dict()) + self.assertRaises(TypeError, load_object, {}) def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') @@ -162,6 +162,15 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test_value' assert os.environ.get('some_test_environ') == 'test' + def test_rel_has_nofollow(self): + assert rel_has_nofollow('ugc nofollow') is True + assert rel_has_nofollow('ugc,nofollow') is True + assert rel_has_nofollow('ugc') is False + assert rel_has_nofollow('nofollow') is True + assert rel_has_nofollow('nofollowfoo') is False + assert rel_has_nofollow('foonofollow') is False + assert rel_has_nofollow('ugc, , nofollow') is True + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 2be38620c..1c85ca353 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,35 +1,116 @@ import unittest +import warnings +from unittest import mock -from scrapy.utils.misc import is_generator_with_return_value +from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value + + +def _indentation_error(*args, **kwargs): + raise IndentationError() + + +def top_level_return_something(): + """ +docstring + """ + url = """ +https://example.org +""" + yield url + return 1 + + +def top_level_return_none(): + """ +docstring + """ + url = """ +https://example.org +""" + yield url + return + + +def generator_that_returns_stuff(): + yield 1 + yield 2 + return 3 class UtilsMiscPy3TestCase(unittest.TestCase): - def test_generators_with_return_statements(self): - def f(): + def test_generators_return_something(self): + def f1(): yield 1 return 2 - def g(): + def g1(): yield 1 - return 'asdf' + return "asdf" - def h(): + def h1(): + yield 1 + + def helper(): + return 0 + + yield helper() + return 2 + + def i1(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return 1 + + assert is_generator_with_return_value(top_level_return_something) + assert is_generator_with_return_value(f1) + assert is_generator_with_return_value(g1) + assert is_generator_with_return_value(h1) + assert is_generator_with_return_value(i1) + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_something) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.top_level_return_something" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.f1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.g1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.h1" method is a generator', str(w[0].message)) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i1) + self.assertEqual(len(w), 1) + self.assertIn('The "NoneType.i1" method is a generator', str(w[0].message)) + + def test_generators_return_none(self): + def f2(): yield 1 return None - def i(): + def g2(): yield 1 return - def j(): + def h2(): yield 1 - def k(): + def i2(): yield 1 - yield from g() + yield from generator_that_returns_stuff() - def m(): + def j2(): yield 1 def helper(): @@ -37,20 +118,56 @@ class UtilsMiscPy3TestCase(unittest.TestCase): yield helper() - def n(): - yield 1 + def k2(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return - def helper(): - return 0 + def l2(): + return - yield helper() - return 2 + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f2) + assert not is_generator_with_return_value(g2) + assert not is_generator_with_return_value(h2) + assert not is_generator_with_return_value(i2) + assert not is_generator_with_return_value(j2) # not recursive + assert not is_generator_with_return_value(k2) # not recursive + assert not is_generator_with_return_value(l2) - assert is_generator_with_return_value(f) - assert is_generator_with_return_value(g) - assert not is_generator_with_return_value(h) - assert not is_generator_with_return_value(i) - assert not is_generator_with_return_value(j) - assert not is_generator_with_return_value(k) # not recursive - assert not is_generator_with_return_value(m) - assert is_generator_with_return_value(n) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, j2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, k2) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, l2) + self.assertEqual(len(w), 0) + + @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) + def test_indentation_error(self): + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 1) + self.assertIn('Unable to determine', str(w[0].message)) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3115cc92f..7dec5624a 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -3,10 +3,10 @@ import gc import operator import platform import unittest -from datetime import datetime from itertools import count -from warnings import catch_warnings +from warnings import catch_warnings, filterwarnings +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, WeakKeyCache, get_func_args, to_bytes, to_unicode, @@ -160,7 +160,11 @@ class UtilsPythonTestCase(unittest.TestCase): pass _values = count() - wk = WeakKeyCache(lambda k: next(_values)) + + with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + wk = WeakKeyCache(lambda k: next(_values)) + k = _Weakme() v = wk[k] self.assertEqual(v, wk[k]) @@ -219,12 +223,7 @@ class UtilsPythonTestCase(unittest.TestCase): elif platform.python_implementation() == 'PyPy': self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj']) - - build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y') - if build_date >= datetime(2020, 4, 7): # PyPy 3.6-v7.3.1 - self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) - else: - self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) + self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 7e0049b1d..e9edfee98 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,73 +1,29 @@ import unittest +import warnings +from hashlib import sha1 +from typing import Dict, Mapping, Optional, Tuple, Union +from weakref import WeakKeyDictionary + +import pytest +from w3lib.url import canonicalize_url + from scrapy.http import Request +from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.python import to_bytes from scrapy.utils.request import ( + _deprecated_fingerprint_cache, _fingerprint_cache, + _request_fingerprint_as_bytes, + fingerprint, request_authenticate, request_fingerprint, request_httprepr, ) +from scrapy.utils.test import get_crawler class UtilsRequestTest(unittest.TestCase): - def test_request_fingerprint(self): - r1 = Request("http://www.example.com/query?id=111&cat=222") - r2 = Request("http://www.example.com/query?cat=222&id=111") - self.assertEqual(request_fingerprint(r1), request_fingerprint(r1)) - self.assertEqual(request_fingerprint(r1), request_fingerprint(r2)) - - r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199') - r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') - self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2)) - - # make sure caching is working - self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)]) - - r1 = Request("http://www.example.com/members/offers.html") - r2 = Request("http://www.example.com/members/offers.html") - r2.headers['SESSIONID'] = b"somehash" - self.assertEqual(request_fingerprint(r1), request_fingerprint(r2)) - - r1 = Request("http://www.example.com/") - r2 = Request("http://www.example.com/") - r2.headers['Accept-Language'] = b'en' - r3 = Request("http://www.example.com/") - r3.headers['Accept-Language'] = b'en' - r3.headers['SESSIONID'] = b"somehash" - - self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3)) - - self.assertEqual(request_fingerprint(r1), - request_fingerprint(r1, include_headers=['Accept-Language'])) - - self.assertNotEqual( - request_fingerprint(r1), - request_fingerprint(r2, include_headers=['Accept-Language'])) - - self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']), - request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language'])) - - r1 = Request("http://www.example.com/test.html") - r2 = Request("http://www.example.com/test.html#fragment") - self.assertEqual(request_fingerprint(r1), request_fingerprint(r2)) - self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True)) - self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True)) - self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True)) - - r1 = Request("http://www.example.com") - r2 = Request("http://www.example.com", method='POST') - r3 = Request("http://www.example.com", method='POST', body=b'request body') - - self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2)) - self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3)) - - # cached fingerprint must be cleared on request copy - r1 = Request("http://www.example.com") - fp1 = request_fingerprint(r1) - r2 = r1.replace(url="http://www.example.com/other") - fp2 = request_fingerprint(r2) - self.assertNotEqual(fp1, fp2) - def test_request_authenticate(self): r = Request("http://www.example.com") request_authenticate(r, 'someuser', 'somepass') @@ -93,5 +49,632 @@ class UtilsRequestTest(unittest.TestCase): request_httprepr(Request("ftp://localhost/tmp/foo.txt")) +class FingerprintTest(unittest.TestCase): + maxDiff = None + + function = staticmethod(fingerprint) + cache: Union[ + "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]", + "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]", + ] = _fingerprint_cache + default_cache_key = (None, False) + known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( + ( + Request("http://example.org"), + b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD', + {}, + ), + ( + Request("https://example.org"), + b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l', + {}, + ), + ( + Request("https://example.org?a"), + b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c', + {}, + ), + ( + Request("https://example.org?a=b"), + b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17', + {}, + ), + ( + Request("https://example.org?a=b&a"), + b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96', + {}, + ), + ( + Request("https://example.org?a=b&a=c"), + b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef', + {}, + ), + ( + Request("https://example.org", method='POST'), + b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d', + {}, + ), + ( + Request("https://example.org", body=b'a'), + b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_', + {}, + ), + ( + Request("https://example.org", method='POST', body=b'a'), + b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7', + {}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l', + {}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.', + {'include_headers': ['A']}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f', + {'keep_fragments': True}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11', + {'include_headers': ['A'], 'keep_fragments': True}, + ), + ( + Request("https://example.org/ab"), + b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6', + {}, + ), + ( + Request("https://example.org/a", body=b'b'), + b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5', + {}, + ), + ) + + def test_query_string_key_order(self): + r1 = Request("http://www.example.com/query?id=111&cat=222") + r2 = Request("http://www.example.com/query?cat=222&id=111") + self.assertEqual(self.function(r1), self.function(r1)) + self.assertEqual(self.function(r1), self.function(r2)) + + def test_query_string_key_without_value(self): + r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199') + r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') + self.assertNotEqual(self.function(r1), self.function(r2)) + + def test_caching(self): + r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') + self.assertEqual( + self.function(r1), + self.cache[r1][self.default_cache_key] + ) + + def test_header(self): + r1 = Request("http://www.example.com/members/offers.html") + r2 = Request("http://www.example.com/members/offers.html") + r2.headers['SESSIONID'] = b"somehash" + self.assertEqual(self.function(r1), self.function(r2)) + + def test_headers(self): + r1 = Request("http://www.example.com/") + r2 = Request("http://www.example.com/") + r2.headers['Accept-Language'] = b'en' + r3 = Request("http://www.example.com/") + r3.headers['Accept-Language'] = b'en' + r3.headers['SESSIONID'] = b"somehash" + + self.assertEqual(self.function(r1), self.function(r2), self.function(r3)) + + self.assertEqual(self.function(r1), + self.function(r1, include_headers=['Accept-Language'])) + + self.assertNotEqual( + self.function(r1), + self.function(r2, include_headers=['Accept-Language'])) + + self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']), + self.function(r3, include_headers=['SESSIONID', 'Accept-Language'])) + + def test_fragment(self): + r1 = Request("http://www.example.com/test.html") + r2 = Request("http://www.example.com/test.html#fragment") + self.assertEqual(self.function(r1), self.function(r2)) + self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True)) + self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True)) + self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True)) + + def test_method_and_body(self): + r1 = Request("http://www.example.com") + r2 = Request("http://www.example.com", method='POST') + r3 = Request("http://www.example.com", method='POST', body=b'request body') + + self.assertNotEqual(self.function(r1), self.function(r2)) + self.assertNotEqual(self.function(r2), self.function(r3)) + + def test_request_replace(self): + # cached fingerprint must be cleared on request copy + r1 = Request("http://www.example.com") + fp1 = self.function(r1) + r2 = r1.replace(url="http://www.example.com/other") + fp2 = self.function(r2) + self.assertNotEqual(fp1, fp2) + + def test_part_separation(self): + # An old implementation used to serialize request data in a way that + # would put the body right after the URL. + r1 = Request("http://www.example.com/foo") + fp1 = self.function(r1) + r2 = Request("http://www.example.com/f", body=b'oo') + fp2 = self.function(r2) + self.assertNotEqual(fp1, fp2) + + def test_hashes(self): + """Test hardcoded hashes, to make sure future changes to not introduce + backward incompatibilities.""" + actual = [ + self.function(request, **kwargs) + for request, _, kwargs in self.known_hashes + ] + expected = [ + _fingerprint + for _, _fingerprint, _ in self.known_hashes + ] + self.assertEqual(actual, expected) + + +class RequestFingerprintTest(FingerprintTest): + function = staticmethod(request_fingerprint) + cache = _deprecated_fingerprint_cache + known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( + ( + Request("http://example.org"), + 'b2e5245ef826fd9576c93bd6e392fce3133fab62', + {}, + ), + ( + Request("https://example.org"), + 'bd10a0a89ea32cdee77917320f1309b0da87e892', + {}, + ), + ( + Request("https://example.org?a"), + '2fb7d48ae02f04b749f40caa969c0bc3c43204ce', + {}, + ), + ( + Request("https://example.org?a=b"), + '42e5fe149b147476e3f67ad0670c57b4cc57856a', + {}, + ), + ( + Request("https://example.org?a=b&a"), + 'd23a9787cb56c6375c2cae4453c5a8c634526942', + {}, + ), + ( + Request("https://example.org?a=b&a=c"), + '9a18a7a8552a9182b7f1e05d33876409e421e5c5', + {}, + ), + ( + Request("https://example.org", method='POST'), + 'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6', + {}, + ), + ( + Request("https://example.org", body=b'a'), + '4bb136e54e715a4ea7a9dd1101831765d33f2d60', + {}, + ), + ( + Request("https://example.org", method='POST', body=b'a'), + '6c6595374a304b293be762f7b7be3f54e9947c65', + {}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + 'bd10a0a89ea32cdee77917320f1309b0da87e892', + {}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + '515b633cb3ca502a33a9d8c890e889ec1e425e65', + {'include_headers': ['A']}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + '505c96e7da675920dfef58725e8c957dfdb38f47', + {'keep_fragments': True}, + ), + ( + Request("https://example.org#a", headers={'A': b'B'}), + 'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da', + {'include_headers': ['A'], 'keep_fragments': True}, + ), + ( + Request("https://example.org/ab"), + '4e2870fee58582d6f81755e9b8fdefe3cba0c951', + {}, + ), + ( + Request("https://example.org/a", body=b'b'), + '4e2870fee58582d6f81755e9b8fdefe3cba0c951', + {}, + ), + ) + + @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True) + def test_part_separation(self): + super().test_part_separation() + + def test_deprecation_default_parameters(self): + with pytest.warns(ScrapyDeprecationWarning) as warnings: + self.function(Request("http://www.example.com")) + messages = [str(warning.message) for warning in warnings] + self.assertTrue( + any( + 'Call to deprecated function' in message + for message in messages + ) + ) + self.assertFalse(any('non-default' in message for message in messages)) + + def test_deprecation_non_default_parameters(self): + with pytest.warns(ScrapyDeprecationWarning) as warnings: + self.function(Request("http://www.example.com"), keep_fragments=True) + messages = [str(warning.message) for warning in warnings] + self.assertTrue( + any( + 'Call to deprecated function' in message + for message in messages + ) + ) + self.assertTrue(any('non-default' in message for message in messages)) + + +class RequestFingerprintAsBytesTest(FingerprintTest): + function = staticmethod(_request_fingerprint_as_bytes) + cache = _deprecated_fingerprint_cache + known_hashes = RequestFingerprintTest.known_hashes + + def test_caching(self): + r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') + self.assertEqual( + self.function(r1), + bytes.fromhex(self.cache[r1][self.default_cache_key]) + ) + + @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True) + def test_part_separation(self): + super().test_part_separation() + + def test_hashes(self): + actual = [ + self.function(request, **kwargs) + for request, _, kwargs in self.known_hashes + ] + expected = [ + bytes.fromhex(_fingerprint) + for _, _fingerprint, _ in self.known_hashes + ] + self.assertEqual(actual, expected) + + +_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary() + + +def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False): + if include_headers: + include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) + cache = _fingerprint_cache_2_6.setdefault(request, {}) + cache_key = (include_headers, keep_fragments) + if cache_key not in cache: + fp = sha1() + fp.update(to_bytes(request.method)) + fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))) + fp.update(request.body or b'') + if include_headers: + for hdr in include_headers: + if hdr in request.headers: + fp.update(hdr) + for v in request.headers.getlist(hdr): + fp.update(v) + cache[cache_key] = fp.hexdigest() + return cache[cache_key] + + +REQUEST_OBJECTS_TO_TEST = ( + Request("http://www.example.com/"), + Request("http://www.example.com/query?id=111&cat=222"), + Request("http://www.example.com/query?cat=222&id=111"), + Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'), + Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'), + Request("http://www.example.com/members/offers.html"), + Request( + "http://www.example.com/members/offers.html", + headers={'SESSIONID': b"somehash"}, + ), + Request( + "http://www.example.com/", + headers={'Accept-Language': b"en"}, + ), + Request( + "http://www.example.com/", + headers={ + 'Accept-Language': b"en", + 'SESSIONID': b"somehash", + }, + ), + Request("http://www.example.com/test.html"), + Request("http://www.example.com/test.html#fragment"), + Request("http://www.example.com", method='POST'), + Request("http://www.example.com", method='POST', body=b'request body'), +) + + +class BackwardCompatibilityTestCase(unittest.TestCase): + + def test_function_backward_compatibility(self): + include_headers_to_test = ( + None, + ['Accept-Language'], + ['accept-language', 'sessionid'], + ['SESSIONID', 'Accept-Language'], + ) + for request_object in REQUEST_OBJECTS_TO_TEST: + for include_headers in include_headers_to_test: + for keep_fragments in (False, True): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + fp = request_fingerprint( + request_object, + include_headers=include_headers, + keep_fragments=keep_fragments, + ) + old_fp = request_fingerprint_2_6( + request_object, + include_headers=include_headers, + keep_fragments=keep_fragments, + ) + self.assertEqual(fp, old_fp) + + def test_component_backward_compatibility(self): + for request_object in REQUEST_OBJECTS_TO_TEST: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + crawler = get_crawler(prevent_warnings=False) + fp = crawler.request_fingerprinter.fingerprint(request_object) + old_fp = request_fingerprint_2_6(request_object) + self.assertEqual(fp.hex(), old_fp) + + def test_custom_component_backward_compatibility(self): + """Tests that the backward-compatible request fingerprinting class featured + in the documentation is indeed backward compatible and does not cause a + warning to be logged.""" + + class RequestFingerprinter: + + cache = WeakKeyDictionary() + + def fingerprint(self, request): + if request not in self.cache: + fp = sha1() + fp.update(to_bytes(request.method)) + fp.update(to_bytes(canonicalize_url(request.url))) + fp.update(request.body or b'') + self.cache[request] = fp.digest() + return self.cache[request] + + for request_object in REQUEST_OBJECTS_TO_TEST: + with warnings.catch_warnings() as logged_warnings: + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + } + crawler = get_crawler(settings_dict=settings) + fp = crawler.request_fingerprinter.fingerprint(request_object) + old_fp = request_fingerprint_2_6(request_object) + self.assertEqual(fp.hex(), old_fp) + self.assertFalse(logged_warnings) + + +class RequestFingerprinterTestCase(unittest.TestCase): + + def test_default_implementation(self): + with warnings.catch_warnings(record=True) as logged_warnings: + crawler = get_crawler(prevent_warnings=False) + request = Request('https://example.com') + self.assertEqual( + crawler.request_fingerprinter.fingerprint(request), + _request_fingerprint_as_bytes(request), + ) + self.assertTrue(logged_warnings) + + def test_deprecated_implementation(self): + settings = { + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION', + } + with warnings.catch_warnings(record=True) as logged_warnings: + crawler = get_crawler(settings_dict=settings) + request = Request('https://example.com') + self.assertEqual( + crawler.request_fingerprinter.fingerprint(request), + _request_fingerprint_as_bytes(request), + ) + self.assertTrue(logged_warnings) + + def test_recommended_implementation(self): + settings = { + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + } + with warnings.catch_warnings(record=True) as logged_warnings: + crawler = get_crawler(settings_dict=settings) + request = Request('https://example.com') + self.assertEqual( + crawler.request_fingerprinter.fingerprint(request), + fingerprint(request), + ) + self.assertFalse(logged_warnings) + + def test_unknown_implementation(self): + settings = { + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5', + } + with self.assertRaises(ValueError): + get_crawler(settings_dict=settings) + + +class CustomRequestFingerprinterTestCase(unittest.TestCase): + + def test_include_headers(self): + + class RequestFingerprinter: + + def fingerprint(self, request): + return fingerprint(request, include_headers=['X-ID']) + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + } + crawler = get_crawler(settings_dict=settings) + + r1 = Request("http://www.example.com", headers={'X-ID': '1'}) + fp1 = crawler.request_fingerprinter.fingerprint(r1) + r2 = Request("http://www.example.com", headers={'X-ID': '2'}) + fp2 = crawler.request_fingerprinter.fingerprint(r2) + self.assertNotEqual(fp1, fp2) + + def test_dont_canonicalize(self): + + class RequestFingerprinter: + cache = WeakKeyDictionary() + + def fingerprint(self, request): + if request not in self.cache: + fp = sha1() + fp.update(to_bytes(request.url)) + self.cache[request] = fp.digest() + return self.cache[request] + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + } + crawler = get_crawler(settings_dict=settings) + + r1 = Request("http://www.example.com?a=1&a=2") + fp1 = crawler.request_fingerprinter.fingerprint(r1) + r2 = Request("http://www.example.com?a=2&a=1") + fp2 = crawler.request_fingerprinter.fingerprint(r2) + self.assertNotEqual(fp1, fp2) + + def test_meta(self): + + class RequestFingerprinter: + + def fingerprint(self, request): + if 'fingerprint' in request.meta: + return request.meta['fingerprint'] + return fingerprint(request) + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + } + crawler = get_crawler(settings_dict=settings) + + r1 = Request("http://www.example.com") + fp1 = crawler.request_fingerprinter.fingerprint(r1) + r2 = Request("http://www.example.com", meta={'fingerprint': 'a'}) + fp2 = crawler.request_fingerprinter.fingerprint(r2) + r3 = Request("http://www.example.com", meta={'fingerprint': 'a'}) + fp3 = crawler.request_fingerprinter.fingerprint(r3) + r4 = Request("http://www.example.com", meta={'fingerprint': 'b'}) + fp4 = crawler.request_fingerprinter.fingerprint(r4) + self.assertNotEqual(fp1, fp2) + self.assertNotEqual(fp1, fp4) + self.assertNotEqual(fp2, fp4) + self.assertEqual(fp2, fp3) + + def test_from_crawler(self): + + class RequestFingerprinter: + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def __init__(self, crawler): + self._fingerprint = crawler.settings['FINGERPRINT'] + + def fingerprint(self, request): + return self._fingerprint + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + 'FINGERPRINT': b'fingerprint', + } + crawler = get_crawler(settings_dict=settings) + + request = Request("http://www.example.com") + fingerprint = crawler.request_fingerprinter.fingerprint(request) + self.assertEqual(fingerprint, settings['FINGERPRINT']) + + def test_from_settings(self): + + class RequestFingerprinter: + + @classmethod + def from_settings(cls, settings): + return cls(settings) + + def __init__(self, settings): + self._fingerprint = settings['FINGERPRINT'] + + def fingerprint(self, request): + return self._fingerprint + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + 'FINGERPRINT': b'fingerprint', + } + crawler = get_crawler(settings_dict=settings) + + request = Request("http://www.example.com") + fingerprint = crawler.request_fingerprinter.fingerprint(request) + self.assertEqual(fingerprint, settings['FINGERPRINT']) + + def test_from_crawler_and_settings(self): + + class RequestFingerprinter: + + # This method is ignored due to the presence of from_crawler + @classmethod + def from_settings(cls, settings): + return cls(settings) + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def __init__(self, crawler): + self._fingerprint = crawler.settings['FINGERPRINT'] + + def fingerprint(self, request): + return self._fingerprint + + settings = { + 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter, + 'FINGERPRINT': b'fingerprint', + } + crawler = get_crawler(settings_dict=settings) + + request = Request("http://www.example.com") + fingerprint = crawler.request_fingerprinter.fingerprint(request) + self.assertEqual(fingerprint, settings['FINGERPRINT']) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d6f4c0bb5..0a09f6109 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(response_status_message(200), '200 OK') self.assertEqual(response_status_message(404), '404 Not Found') self.assertEqual(response_status_message(573), "573 Unknown Status") + + def test_inject_base_url(self): + url = "http://www.example.com" + + def check_base_url(burl): + path = urlparse(burl).path + if not os.path.exists(path): + path = burl.replace('file://', '') + with open(path, "rb") as f: + bbody = f.read() + self.assertEqual(bbody.count(b''), 1) + return True + + r1 = HtmlResponse(url, body=b""" + + Dummy +

Hello world.

+ """) + r2 = HtmlResponse(url, body=b""" + + Dummy + Hello world. + """) + r3 = HtmlResponse(url, body=b""" + + Dummy + +
Hello header
+

Hello world.

+ + """) + r4 = HtmlResponse(url, body=b""" + + + Dummy +

Hello world.

+ """) + r5 = HtmlResponse(url, body=b""" + + + + Standard head + +

Hello world.

+ """) + + assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" + assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head" + assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag" + assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment" + assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment" diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index b66588efb..a36e7bc97 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,11 +1,11 @@ import asyncio +from pydispatch import dispatcher from pytest import mark from testfixtures import LogCapture -from twisted.trial import unittest -from twisted.python.failure import Failure from twisted.internet import defer, reactor -from pydispatch import dispatcher +from twisted.python.failure import Failure +from twisted.trial import unittest from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.test import get_from_asyncio_queue @@ -68,6 +68,7 @@ class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): return d +@mark.usefixtures('reactor_pytest') class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): async def ok_handler(self, arg, handlers_called): @@ -76,6 +77,9 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" + def test_send_catch_log(self): + return super().test_send_catch_log() + @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -86,6 +90,9 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") + def test_send_catch_log(self): + return super().test_send_catch_log() + class SendCatchLogTest2(unittest.TestCase): diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5ff2e41ef..1d5e63363 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -36,7 +36,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): self.assertEqual(result.read().decode('utf8'), rendered) os.remove(render_path) - assert not os.path.exists(render_path) # Failure of test iself + assert not os.path.exists(render_path) # Failure of test itself if '__main__' == __name__: diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a60181a3a..0d5827339 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -18,14 +18,6 @@ except ImportError: from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks -from twisted.web.test.test_webclient import ( - ForeverTakingResource, - ErrorResource, - NoLengthResource, - HostHeaderResource, - PayloadResource, - BrokenDownloadResource, -) from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory @@ -33,7 +25,15 @@ from scrapy.http import Request, Headers from scrapy.settings import Settings from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes, to_unicode -from tests.mockserver import ssl_context_factory +from tests.mockserver import ( + BrokenDownloadResource, + ErrorResource, + ForeverTakingResource, + HostHeaderResource, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt new file mode 100644 index 000000000..2a335e533 --- /dev/null +++ b/tests/upper-constraints.txt @@ -0,0 +1,17 @@ +# Request the latest known version or newer of some dependencies to prevent the +# pip dependency resolver from spending too much time backtracking. +attrs>=20.2.0 +Automat>=0.8.0 +botocore>=1.20.30 +itemadapter>=0.1.1 +itemloaders>=1.0.3 +lxml>=4.6.1 +parsel>=1.5.2 +Pillow>=8.0.1 +pyOpenSSL>=17.5 # mitmproxy 4.0.4 +pytest>=6.2.1 +pytest-twisted>=1.13.1 +service_identity>=17.0.0 +six>=1.14.0 +sybil>=2.0.0 +Twisted>=19.10.0 diff --git a/tox.ini b/tox.ini index ea356c56a..ab8a715c2 100644 --- a/tox.ini +++ b/tox.ini @@ -9,31 +9,45 @@ minversion = 1.7.0 [testenv] deps = - -ctests/constraints.txt - -rtests/requirements-py3.txt + -rtests/requirements.txt + # mitmproxy does not support PyPy + # Python 3.9+ requires mitmproxy >= 5.3.0 + # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 + #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' + # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 + mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' + # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) + markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy' # Extras botocore>=1.4.87 - Pillow>=4.0.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN GCS_TEST_FILE_URI GCS_PROJECT_ID +#allow tox virtualenv to upgrade pip/wheel/setuptools +download = true commands = - py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} +install_command = + pip install -U -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] basepython = python3 deps = - mypy==0.780 + lxml-stubs==0.2.0 + mypy==0.910 + types-pyOpenSSL==20.0.3 + types-setuptools==57.0.0 commands = - mypy {posargs: scrapy tests} + mypy --show-error-codes {posargs: scrapy tests} [testenv:security] basepython = python3 deps = - bandit + bandit==1.7.3 commands = bandit -r -c .bandit.yml {posargs:scrapy} @@ -41,83 +55,105 @@ commands = basepython = python3 deps = {[testenv]deps} + # Twisted[http2] is required to import some files + Twisted[http2]>=17.9.0 pytest-flake8 + flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 commands = - py.test --flake8 {posargs:docs scrapy tests} + pytest --flake8 {posargs:docs scrapy tests} [testenv:pylint] basepython = python3 deps = - {[testenv]deps} - # Optional dependencies - boto - reppy - robotexclusionrulesparser - # Test dependencies - pylint + {[testenv:extra-deps]deps} + pylint==2.12.2 commands = pylint conftest.py docs extras scrapy setup.py tests [pinned] deps = - -ctests/constraints.txt - cryptography==2.0 + cryptography==2.8 cssselect==0.9.1 + h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - PyDispatcher==2.0.5 - pyOpenSSL==16.2.0 + pyOpenSSL==19.1.0 queuelib==1.4.2 service_identity==16.0.0 - Twisted==17.9.0 + Twisted[http2]==18.9.0 w3lib==1.17.0 - zope.interface==4.1.3 - -rtests/requirements-py3.txt + zope.interface==5.1.0 + lxml==4.3.0 + -rtests/requirements.txt + + # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies + # above, hence we do not install it in pinned environments at the moment + # Extras botocore==1.4.87 google-cloud-storage==1.29.0 - Pillow==4.0.0 + Pillow==7.1.0 +setenv = + _SCRAPY_PINNED=true +install_command = + pip install -U {opts} {packages} [testenv:pinned] deps = {[pinned]deps} - lxml==3.5.0 + PyDispatcher==2.0.5 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:windows-pinned] basepython = python3 deps = {[pinned]deps} - # First lxml version that includes a Windows wheel for Python 3.6, so we do - # not need to build lxml from sources in a CI Windows job: - lxml==3.8.0 + PyDispatcher==2.0.5 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:extra-deps] deps = {[testenv]deps} + boto + google-cloud-storage + # Twisted[http2] currently forces old mitmproxy because of h2 version + # restrictions in their deps, so we need to pin old markupsafe here too. + markupsafe < 2.1.0 reppy robotexclusionrulesparser + Pillow>=4.0.0 + Twisted[http2]>=17.9.0 [testenv:asyncio] commands = {[testenv]commands} --reactor=asyncio [testenv:asyncio-pinned] -commands = {[testenv:asyncio]commands} deps = {[testenv:pinned]deps} +commands = {[testenv:asyncio]commands} +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:pypy3] basepython = pypy3 commands = - py.test {posargs:--durations=10 docs scrapy tests} + pytest {posargs:--durations=10 docs scrapy tests} [testenv:pypy3-pinned] basepython = {[testenv:pypy3]basepython} -commands = {[testenv:pypy3]commands} deps = {[pinned]deps} - lxml==4.0.0 PyPyDispatcher==2.1.0 +commands = {[testenv:pypy3]commands} +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [docs] changedir = docs