diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..98a74f8ce --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,31 @@ + diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml new file mode 100644 index 000000000..160b39488 --- /dev/null +++ b/.github/workflows/auto-close-llm-pr.yml @@ -0,0 +1,48 @@ +name: Auto-close LLM PRs +on: + pull_request_target: + types: [opened] +permissions: + contents: read + pull-requests: write +jobs: + close-llm-pr: + name: Close PR if marked as LLM-written + runs-on: ubuntu-latest + steps: + - name: Check PR body and close if LLM-written + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const marker = "This PR was written entirely using an LLM"; + const { owner, repo } = context.repo; + const prNumber = context.payload.pull_request && context.payload.pull_request.number; + if (!prNumber) { + console.log('No pull request number found in context; exiting.'); + return; + } + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); + const body = pr.body || ""; + if (body.includes(marker)) { + if (pr.state === 'closed') { + console.log(`PR #${prNumber} already closed.`); + return; + } + await github.rest.issues.addLabels({ + owner, + repo, + issue_number: prNumber, + labels: ['spam'] + }); + await github.rest.issues.createComment({ + owner, + repo, + issue_number: prNumber, + body: "Closing this PR because it contains the disclosure: \"This PR was written entirely using an LLM\"." + }); + await github.rest.pulls.update({ owner, repo, pull_number: prNumber, state: 'closed' }); + console.log(`Closed PR #${prNumber} because marker was found.`); + } else { + console.log(`Marker not found in PR #${prNumber}; nothing to do.`); + } diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d3715caf2..ed2388a59 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -17,19 +17,23 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: pylint - python-version: "3.10" env: - TOXENV: typing + TOXENV: mypy - python-version: "3.10" env: - TOXENV: typing-tests - - python-version: "3.13" # Keep in sync with .readthedocs.yml + TOXENV: mypy-tests + # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version. + - python-version: "3.14" env: TOXENV: docs - python-version: "3.13" + env: + TOXENV: docs-tests + - python-version: "3.14" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ad327e465..7779bbb6b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: - python-version: "3.13" + python-version: "3.14" - run: | python -m pip install --upgrade build python -m build diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 3adf3de38..0409b3ef2 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -18,7 +18,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + env: + - TOXENV: py + include: + - python-version: '3.14' + env: + TOXENV: no-reactor steps: - uses: actions/checkout@v6 @@ -29,9 +35,10 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run tests + env: ${{ matrix.env }} run: | pip install -U tox - tox -e py + tox - name: Upload coverage report uses: codecov/codecov-action@v5 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index c07cd49f0..f51dd9799 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -31,46 +31,55 @@ jobs: - python-version: "3.13" env: TOXENV: py - - python-version: "3.13" + - python-version: "3.14" + env: + TOXENV: py + - python-version: "3.14" env: TOXENV: default-reactor - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: no-reactor - - python-version: pypy3.11 + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3 - # pinned deps + # min deps - python-version: "3.10.19" env: - TOXENV: pinned + TOXENV: min - python-version: "3.10.19" env: - TOXENV: default-reactor-pinned + TOXENV: min-default-reactor - python-version: "3.10.19" env: - TOXENV: no-reactor-pinned - - python-version: pypy3.11 + TOXENV: min-no-reactor + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 env: - TOXENV: pypy3-pinned + TOXENV: min-pypy3 - python-version: "3.10.19" env: - TOXENV: extra-deps-pinned + TOXENV: min-extra-deps - python-version: "3.10.19" env: - TOXENV: botocore-pinned + TOXENV: min-botocore - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: extra-deps - - python-version: pypy3.11 + - python-version: "3.14" + env: + TOXENV: no-reactor-extra-deps + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3-extra-deps - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: botocore - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: mitmproxy @@ -83,7 +92,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') + if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 48aa56e15..f413782bc 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -31,22 +31,25 @@ jobs: - python-version: "3.13" env: TOXENV: py - - python-version: "3.13" + - python-version: "3.14" + env: + TOXENV: py + - python-version: "3.14" env: TOXENV: default-reactor - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: no-reactor - # pinned deps + # min deps - python-version: "3.10.11" env: - TOXENV: pinned + TOXENV: min - python-version: "3.10.11" env: - TOXENV: extra-deps-pinned + TOXENV: min-extra-deps - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: extra-deps diff --git a/.gitignore b/.gitignore index 4100bcd97..5e52ecf1e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ *.pyc _trial_temp* dropin.cache -docs/build +docs/_build *egg-info .tox/ venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2bf1be0c..754bd6f9c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,3 +26,7 @@ repos: rev: v1.0.2 hooks: - id: sphinx-lint +- repo: https://github.com/scrapy/sphinx-scrapy + rev: 0.8.8 + hooks: + - id: sphinx-scrapy diff --git a/.readthedocs.yml b/.readthedocs.yml index 23e4cabea..a2773dcf2 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,17 +1,10 @@ version: 2 -formats: all -sphinx: - configuration: docs/conf.py - fail_on_warning: true - build: os: ubuntu-24.04 tools: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.13" # Keep in sync with .github/workflows/checks.yml - -python: - install: - - requirements: docs/requirements.txt - - path: . + python: "3.14" + commands: + - pip install tox + - tox -e docs + - mkdir -p $READTHEDOCS_OUTPUT/html + - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..24a426d36 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,6 @@ +cff-version: 1.2.0 +message: If you use Scrapy in published research, please cite it as below. +title: Scrapy +authors: + - name: Scrapy contributors +url: https://scrapy.org diff --git a/SECURITY.md b/SECURITY.md index db67dea5d..49cafdf9e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,8 +4,8 @@ | Version | Supported | | ------- | ------------------ | -| 2.14.x | :white_check_mark: | -| < 2.14.x | :x: | +| 2.16.x | :white_check_mark: | +| < 2.16.x | :x: | ## Reporting a Vulnerability diff --git a/conftest.py b/conftest.py index d49901a7c..532f83f56 100644 --- a/conftest.py +++ b/conftest.py @@ -11,6 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy from scrapy.utils.reactorless import install_reactor_import_hook from tests.keys import generate_keys from tests.mockserver.http import MockServer +from tests.mockserver.mitm_proxy import MitmProxy if TYPE_CHECKING: from collections.abc import Generator @@ -23,16 +24,13 @@ def _py_files(folder): collect_ignore = [ # may need extra deps "docs/_ext", - # not a test, but looks like a test - "scrapy/utils/testproc.py", - "scrapy/utils/testsite.py", - # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerProcessSubprocess *_py_files("tests/AsyncCrawlerProcess"), - # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerRunnerSubprocess *_py_files("tests/AsyncCrawlerRunner"), - # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), - # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), ] @@ -75,6 +73,24 @@ def mockserver() -> Generator[MockServer]: yield mockserver +@pytest.fixture # function scope because it modifies os.environ +def proxy_server( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Generator[str]: + kind = request.param + proxy = MitmProxy(mode="socks5" if kind == "socks5" else None) + url = proxy.start() + if kind == "https": + url = url.replace("http://", "https://") + monkeypatch.setenv("http_proxy", url) + monkeypatch.setenv("https_proxy", url) + + try: + yield kind + finally: + proxy.stop() + + @pytest.fixture(scope="session") def reactor_pytest(request) -> str: return request.config.getoption("--reactor") diff --git a/docs/README.rst b/docs/README.rst index 36dd5aea4..4d2236b53 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -65,4 +65,4 @@ To compile the documentation to HTML run the following command:: tox -e docs -Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir. +Documentation will be generated inside the ``docs/_build/all`` dir. diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 776afaa6c..edb91bfb9 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -77,6 +77,25 @@ def make_setting_element( return item +def make_setting_markdown_item( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> str: + uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"]) + if uri.startswith("#"): + target = f"#{setting_data['refid']}" + else: + target = f"{uri}#{setting_data['refid']}" + return f"* [{setting_data['setting_name']}]({target})" + + +def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]: + return [ + d + for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] + if fromdocname != d["docname"] + ] + + def replace_settingslist_nodes( app: Sphinx, doctree: document, fromdocname: str ) -> None: @@ -87,13 +106,29 @@ def replace_settingslist_nodes( settings_list.extend( [ make_setting_element(d, app, fromdocname) - for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] - if fromdocname != d["docname"] + for d in _iter_sorted_settings(env, fromdocname) ] ) node.replace_self(settings_list) +def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None: + builder = translator.builder + env = builder.env + fromdocname = getattr(builder, "current_doc_name", env.docname) + lines = [ + make_setting_markdown_item(setting_data, builder.app, fromdocname) + for setting_data in _iter_sorted_settings(env, fromdocname) + ] + if lines: + translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2) + raise nodes.SkipNode + + +def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None: + return None + + def source_role( name, rawtext, text: str, lineno, inliner, options=None, content=None ) -> tuple[list[Any], list[Any]]: @@ -126,34 +161,22 @@ def rev_role( return [node], [] -def setup(app: Sphinx) -> None: - app.add_crossref_type( - directivename="setting", - rolename="setting", - indextemplate="pair: %s; setting", - ) - app.add_crossref_type( - directivename="signal", - rolename="signal", - indextemplate="pair: %s; signal", - ) - app.add_crossref_type( - directivename="command", - rolename="command", - indextemplate="pair: %s; command", - ) - app.add_crossref_type( - directivename="reqmeta", - rolename="reqmeta", - indextemplate="pair: %s; reqmeta", - ) +def setup(app: Sphinx) -> dict[str, Any]: app.add_role("source", source_role) app.add_role("commit", commit_role) app.add_role("issue", issue_role) app.add_role("rev", rev_role) - app.add_node(SettingslistNode) + app.add_node( + SettingslistNode, + markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown), + singlemarkdown=( + visit_settingslist_node_markdown, + depart_settingslist_node_markdown, + ), + ) app.add_directive("settingslist", SettingsListDirective) app.connect("doctree-read", collect_scrapy_settings_refs) app.connect("doctree-resolved", replace_settingslist_nodes) + return {"parallel_read_safe": True} diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py index d7a3fb514..e342e92cf 100644 --- a/docs/_ext/scrapyfixautodoc.py +++ b/docs/_ext/scrapyfixautodoc.py @@ -3,16 +3,19 @@ Must be included after 'sphinx.ext.autodoc'. Fixes unwanted 'alias of' behavior. https://github.com/sphinx-doc/sphinx/issues/4422 """ +from typing import Any + # pylint: disable=import-error from sphinx.application import Sphinx def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool: if not skip: - # autodocs was generating a text "alias of" for the following members + # autodoc was generating the text "alias of" for the following members return name in {"default_item_class", "default_selector_class"} return skip -def setup(app: Sphinx) -> None: +def setup(app: Sphinx) -> dict[str, Any]: app.connect("autodoc-skip-member", maybe_skip_member) + return {"parallel_read_safe": True} diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 6ec565e24..29394799b 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -1,6 +1,6 @@ {% extends "!layout.html" %} -{# Overriden to include a link to scrapy.org, not just to the docs root #} +{# Overridden to include a link to scrapy.org, not just to the docs root #} {%- block sidebartitle %} {# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #} diff --git a/docs/conf.py b/docs/conf.py index 9b1db9bb7..b950c4ee2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,11 +28,9 @@ author = "Scrapy developers" extensions = [ "notfound.extension", "scrapydocs", - "sphinx.ext.autodoc", + "sphinx_scrapy", "scrapyfixautodoc", # Must be after "sphinx.ext.autodoc" "sphinx.ext.coverage", - "sphinx.ext.intersphinx", - "sphinx.ext.viewcode", "sphinx_rtd_dark_mode", ] @@ -147,22 +145,26 @@ coverage_ignore_pyobjects = [ # -- Options for the InterSphinx extension ----------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration -intersphinx_mapping = { - "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/latest", None), - "cryptography": ("https://cryptography.io/en/latest/", None), - "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), - "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), - "parsel": ("https://parsel.readthedocs.io/en/latest/", None), - "pytest": ("https://docs.pytest.org/en/latest", None), - "python": ("https://docs.python.org/3", None), - "sphinx": ("https://www.sphinx-doc.org/en/master", None), - "tox": ("https://tox.wiki/en/latest/", None), - "twisted": ("https://docs.twisted.org/en/stable/", None), - "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), - "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), -} intersphinx_disabled_reftypes: Sequence[str] = [] +# sphinx-scrapy --------------------------------------------------------------- + +scrapy_intersphinx_enable = [ + "attrs", + "coverage", + "cryptography", + "cssselect", + "form2request", + "itemloaders", + "parsel", + "pytest", + "scrapy-lint", + "sphinx", + "tox", + "twisted", + "twistedapi", + "w3lib", +] + # -- Other options ------------------------------------------------------------ default_dark_mode = False diff --git a/docs/contributing.rst b/docs/contributing.rst index 3976d34c2..c868a0ac4 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -258,7 +258,7 @@ Scrapy: * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. - See https://docs.github.com/en/get-started/getting-started-with-git/setting-your-username-in-git + See https://docs.github.com/en/get-started/git-basics/setting-your-username-in-git for setup instructions. .. _scrapy-pre-commit: @@ -390,8 +390,7 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users -.. _Scrapy subreddit: https://reddit.com/r/scrapy -.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS +.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/ .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues .. _PEP 257: https://peps.python.org/pep-0257/ diff --git a/docs/faq.rst b/docs/faq.rst index fdf0582de..df90122f5 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -82,10 +82,18 @@ to steal from us! Does Scrapy work with HTTP proxies? ----------------------------------- -Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP -Proxy downloader middleware. See +Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader +middleware. See :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`. +Does Scrapy work with SOCKS proxies? +------------------------------------ + +Yes, when using +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the +handler documentation. + How can I scrape an item with attributes in different pages? ------------------------------------------------------------ @@ -360,7 +368,10 @@ method for this purpose. For example: Does Scrapy support IPv6 addresses? ----------------------------------- -Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. +Yes, but when using +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or +:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to +set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. Note that by doing so, you lose the ability to set a specific timeout for DNS requests (the value of the :setting:`DNS_TIMEOUT` setting is ignored). @@ -418,4 +429,3 @@ See :issue:`2680`. .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 .. _Python standard library modules: https://docs.python.org/3/py-modindex.html .. _Python package: https://pypi.org/ -.. _user agents: https://en.wikipedia.org/wiki/User_agent diff --git a/docs/index.rst b/docs/index.rst index fe61f3513..a46a2ad9f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -128,7 +128,6 @@ Built-in services topics/logging topics/stats - topics/email topics/telnetconsole :doc:`topics/logging` @@ -137,9 +136,6 @@ Built-in services :doc:`topics/stats` Collect statistics about your scraping crawler. -:doc:`topics/email` - Send email notifications when certain events occur. - :doc:`topics/telnetconsole` Inspect a running crawler using a built-in Python console. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 112fbfe04..8d8b50fbf 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -265,7 +265,6 @@ reinstall Twisted with the :code:`tls` extra option:: For details, see `Issue #2473 `_. .. _Python: https://www.python.org/ -.. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: https://lxml.de/index.html .. _parsel: https://pypi.org/project/parsel/ .. _w3lib: https://pypi.org/project/w3lib/ @@ -275,8 +274,7 @@ For details, see `Issue #2473 `_. .. _setuptools: https://pypi.org/pypi/setuptools .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Anaconda: https://docs.anaconda.com/anaconda/ +.. _Anaconda: https://www.anaconda.com/docs/main .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html -.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio .. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index d05e46551..ee91ce7ca 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -150,7 +150,7 @@ The next steps for you are to :ref:`install Scrapy `, a full-blown Scrapy project and `join the community`_. Thanks for your interest! -.. _join the community: https://scrapy.org/community/ +.. _join the community: https://www.scrapy.org/community .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping .. _Amazon Associates Web Services: https://affiliate-program.amazon.com/welcome/ecs .. _Amazon S3: https://aws.amazon.com/s3/ diff --git a/docs/news.rst b/docs/news.rst index ff8e6fb08..b8b976df9 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,17 +3,750 @@ Release notes ============= -Scrapy VERSION (unreleased) ---------------------------- +.. _release-2.16.0: + +Scrapy 2.16.0 (2026-05-19) +-------------------------- + +Highlights: + +- Official support for Python 3.14 + +- Support for Twisted 26.4.0+ + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Increased the minimum versions of the following dependencies: + + - service_identity_: 18.1.0 → 23.1.0 + + (:issue:`7347`) + +- Added support for Twisted 26.4.0+. + (:issue:`7347`, :issue:`7505`, :issue:`7520`) + +- Added support for Python 3.14. + (:issue:`6604`, :issue:`7460`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The following classes and functions, intended for internal use by + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have + been made private: + + - ``scrapy.core.downloader.handlers.http11.ScrapyAgent`` + + - ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint`` + + - ``scrapy.core.downloader.handlers.http11.tunnel_request_data()`` + + - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent`` + + (:issue:`7496`, :issue:`7510`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.FormRequest`` is deprecated. You can use the :doc:`form2request + ` library instead, see :ref:`form`. + (:issue:`6438`) + +- ``scrapy.utils.python.MutableChain`` is deprecated. + (:issue:`7504`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider`, deprecated in + 2.13.0, is removed and no longer called. Use :meth:`~scrapy.Spider.start` + instead, or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for ``process_start_requests()`` methods of :ref:`spider middlewares + `, deprecated in 2.13.0, is removed. Use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for synchronous ``process_spider_output()`` methods of spider + middlewares, deprecated in Scrapy 2.13.0, is removed. You should upgrade + the affected middlewares to have asynchronous ``process_spider_output()`` + methods. + (:issue:`7504`) + +- The ``spider`` arguments of the following methods of + :class:`~scrapy.core.scraper.Scraper`, deprecated in Scrapy 2.13.0, are + removed: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`7487`) + +- HTTP/1.0 support code, deprecated in Scrapy 2.13.0, is removed. This + includes: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - The ``scrapy.core.downloader.webclient`` module. + + - The ``DOWNLOADER_HTTPCLIENTFACTORY`` setting. + + (:issue:`7486`) + +- The following functions, deprecated in Scrapy 2.13.0, are removed, you + should import them from :mod:`w3lib.url` directly instead: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + (:issue:`7487`) + +- The following test-related code, deprecated in Scrapy 2.13.0, is removed: + + - the ``scrapy.utils.testproc`` module + + - the ``scrapy.utils.testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + - ``scrapy.utils.test.TestSpider`` + + (:issue:`7487`) + +- ``scrapy.utils.versions.scrapy_components_versions()``, deprecated in + Scrapy 2.13.0, is removed, you can use + :func:`scrapy.utils.versions.get_versions` instead. + (:issue:`7487`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` and + ``scrapy.utils.url.escape_ajax()``, deprecated in Scrapy 2.13.0, are + removed. + (:issue:`7487`) + +- The ``__init__()`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) now needs to support a keyword-only + ``start_queue_cls`` parameter, not supporting it was deprecated in Scrapy + 2.13.0. + (:issue:`7487`) + +- ``scrapy.spiders.init.InitSpider``, deprecated in Scrapy 2.13.0, is + removed. + (:issue:`7487`) New features ~~~~~~~~~~~~ +- New features and improvements for + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`: + + - Support for proxies. + + - Support for the :reqmeta:`download_latency` meta key. + + - Support for :attr:`Response.certificate + `. + + - Default headers set by the ``httpx`` library are no longer added to + requests. + + (:issue:`7441`, :issue:`7524`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + skips HTTPS proxy certificate verification when the + :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting is set to ``False``. + (:issue:`7496`) + +Improvements +~~~~~~~~~~~~ + +- :func:`time.monotonic` is used instead of :func:`time.time` to calculate + elapsed time in various places. + (:issue:`7377`) + +- Improved extraction of the file extension from the URL in + :class:`~scrapy.pipelines.files.FilesPipeline`. + (:issue:`4225`, :issue:`7414`) + +- Other code refactoring and improvements. + (:issue:`7401`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + raises an exception when a request has an ``https://`` destination and an + ``https://`` proxy, which is not supported by this handler. Previously it + tried to connect to the proxy via HTTP in this case. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now + raises an exception for requests with ``http://`` URLs instead of trying to + connect, which is not supported by this handler. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` no longer + adds the ``:status`` pseudo-header to :attr:`Response.headers + `. + (:issue:`7441`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` removing the ```` + tag when adding the ```` tag. + (:issue:`7459`) + +Documentation +~~~~~~~~~~~~~ + +- Documented that + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + doesn't support HTTPS proxies for HTTPS destinations and that + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` doesn't + support proxies at all. + (:issue:`7496`) + +- Added an example of using + :class:`logging.handlers.TimedRotatingFileHandler` to rotate Scrapy logs. + (:issue:`3628`, :issue:`7501`) + +- Added a ``CITATION.cff`` file. + (:issue:`7502`, :issue:`7519`) + +- Mentioned ``DOWNLOADER_CLIENT_TLS_METHOD`` in :ref:`bans`. + (:issue:`5232`, :issue:`7518`) + +- Other documentation improvements and fixes. + (:issue:`7417`, + :issue:`7463`, + :issue:`7472`, + :issue:`7480`, + :issue:`7489`, + :issue:`7503`, + :issue:`7507`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added tests that connect to https://books.toscrape.com/ to test the + behavior with a real website. These tests are marked with the + ``requires_internet`` pytest mark and can be skipped with e.g. + ``-m 'not requires_internet'`` if you cannot or don't want to run them. + (:issue:`7520`) + +- Type hints improvements and fixes. + (:issue:`7492`, :issue:`7532`) + +- CI and test improvements and fixes. + (:issue:`7441`, :issue:`7466`, :issue:`7491`, :issue:`7496`) + +.. _release-2.15.2: + +Scrapy 2.15.2 (2026-04-28) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Fixed links in https://docs.scrapy.org/llms.txt (:issue:`7467`) + +.. _release-2.15.1: + +Scrapy 2.15.1 (2026-04-23) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Sharing of the SSL context between multiple connections, introduced in + Scrapy 2.15.0, is reverted as it caused problems and wasn't actually + needed. + (:issue:`7445`, :issue:`7450`) + +- Fixed :meth:`scrapy.settings.BaseSettings.getwithbase` failing on keys with + dots that aren't import names. It now works the way it worked before Scrapy + 2.15.0, without trying to match class objects and import path. A separate + method, + :func:`~scrapy.settings.BaseSettings.get_component_priority_dict_with_base`, + was added that does that, and it is now used for :ref:`component priority + dictionaries `. + (:issue:`7426`, :issue:`7449`) + +- Documentation rendering improvements. + (:issue:`7452`, :issue:`7454`) + +.. _release-2.15.0: + +Scrapy 2.15.0 (2026-04-09) +-------------------------- + +Highlights: + +- Experimental support for running without a Twisted reactor + +- Experimental ``httpx``-based download handler + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The built-in HTTP :ref:`download handlers ` now + raise Scrapy-specific exceptions instead of implementation-specific ones, + see :ref:`download-handlers-exceptions`. This can affect user code that + handles downloader exceptions, such as ``process_exception()`` methods of + custom :ref:`downloader middlewares `. + (:issue:`7208`) + +- In order to fix a long-standing bug with handling of asynchronous storages, + the following changes were made to media pipeline classes, which can impact + some of the user code that subclasses them or calls their methods directly: + + - overrides of :meth:`scrapy.pipelines.media.MediaPipeline.media_downloaded` + and :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` can now + return coroutines + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`, + :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` and + :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` now + return coroutines + + (:issue:`2183`, :issue:`6369`, :issue:`7182`) + +- ``Request`` and ``Response`` objects: ``__slots__`` and setter changes: + + - :class:`scrapy.http.Request` and :class:`scrapy.http.Response` now + define ``__slots__``. Assigning arbitrary attributes to instances (for + example, ``response.foo = 1``) will raise ``AttributeError``. Store + per-request/response data in the request/response ``meta`` mapping + instead of attaching new attributes to the objects. + + - If you maintain custom ``Request`` or ``Response`` subclasses that + relied on dynamic instance attributes, either add ``'__dict__'`` to + your subclass ``__slots__`` to allow dynamic attributes, or migrate + per-instance state to ``meta`` or explicit documented attributes. + + - The setters for ``headers``, ``flags`` and ``cookies`` no longer coerce + falsy values into ``None``. For example, ``request.headers = {}`` now + stores an empty :class:`scrapy.http.headers.Headers` instance (not + ``None``), and ``request.flags = []`` remains an empty list instead of + being set to ``None``. Update code that relied on ``is None`` checks or + the previous coercion behaviour. + + (:issue:`7036`, :issue:`7367`, :issue:`7374`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The context factory class set as the value of the + ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is now required to support the + ``method`` argument of ``__init__()``, recommended since Scrapy 1.2.0. + (:issue:`7353`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.mail.MailSender`` is deprecated. Please use :mod:`smtplib`, + :mod:`twisted.mail.smtp` or other 3rd party email libraries. + (:issue:`7249`, :issue:`7263`) + +- The ``scrapy.extensions.statsmailer.StatsMailer`` extension is deprecated. + You can instead implement your own notifications by handling the + :signal:`spider_closed` signal. + (:issue:`7249`, :issue:`7263`) + +- The ``MEMUSAGE_NOTIFY_MAIL`` setting is deprecated. You can instead + implement your own notifications by handling the + :signal:`memusage_warning_reached` and :signal:`spider_closed` signals. + (:issue:`7249`, :issue:`7263`) + +- The ``DNS_RESOLVER`` setting was renamed to :setting:`TWISTED_DNS_RESOLVER` + and the old name is deprecated. + (:issue:`7350`, :issue:`7361`) + +- The ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is deprecated. If you were + using it to switch to + ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory``, please + use the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting instead. If you + cannot use the default context factory for some other reason, please + subclass the :ref:`download handler ` instead. + (:issue:`7352`, :issue:`7379`) + +- ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory`` is + deprecated. You can set the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` + setting to ``True`` instead. + (:issue:`7379`) + +- The following implementation details of the context factory handling code + are deprecated: + + - ``scrapy.core.downloader.contextfactory.AcceptableProtocolsContextFactory`` + + - ``scrapy.core.downloader.contextfactory.load_context_factory_from_settings()`` + + - ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory`` + + - ``scrapy.core.downloader.tls.ScrapyClientTLSOptions`` + + (:issue:`7353`, :issue:`7391`) + +- Passing :class:`str` instead of :class:`bytes` to + :class:`scrapy.utils.sitemap.Sitemap` and + :func:`scrapy.utils.sitemap.sitemap_urls_from_robots` is deprecated. + (:issue:`7007`) + +- ``scrapy.utils.misc.walk_modules()`` is deprecated. You can use + :func:`scrapy.utils.misc.walk_modules_iter` instead. + (:issue:`7388`) + +- ``scrapy.shell.Shell.inthread`` is deprecated. You can use + :attr:`scrapy.shell.Shell.fetch_available` instead to check if + :func:`~scrapy.shell.Shell.fetch` can be used. + (:issue:`7395`) + +- ``scrapy.commands.ScrapyCommand.set_crawler()`` is deprecated. + (:issue:`7276`) + +New features +~~~~~~~~~~~~ + +- Added an *experimental* mode for running Scrapy without installing a + Twisted reactor: set :setting:`TWISTED_REACTOR_ENABLED` to ``False`` to + enable it. This mode has limitations, refer to :ref:`its documentation + ` for details. As long as it's experimental, its + behavior and related features and APIs may change in future Scrapy releases + in a breaking way. + (:issue:`6219`, + :issue:`7185`, + :issue:`7186`, + :issue:`7187`, + :issue:`7188`, + :issue:`7190`, + :issue:`7197`, + :issue:`7199`, + :issue:`7209`, + :issue:`7228`, + :issue:`7355`, + :issue:`7366`, + :issue:`7385`, + :issue:`7395`) + +- Added the :func:`scrapy.utils.reactorless.is_reactorless` function that + checks if there is a running asyncio event loop but no Twisted reactor. + (:issue:`7185`, :issue:`7199`) + +- Changed :func:`scrapy.utils.asyncio.is_asyncio_available` to return + ``True`` if there is a running asyncio loop, even if no Twisted reactor is + installed. + (:issue:`7185`, :issue:`7199`) + +- Added an *experimental* download handler that uses the httpx_ library and + doesn't require a Twisted reactor: + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. As + long as it's experimental, its behavior may change in future Scrapy + releases in a breaking way. + (:issue:`6805`, :issue:`7239`, :issue:`7368`, :issue:`7384`) + + .. _httpx: https://www.python-httpx.org/ + +- Added the :setting:`DOWNLOAD_BIND_ADDRESS` setting as a global counterpart + to the per-request :reqmeta:`bindaddress` meta key. + (:issue:`7266`, :issue:`7283`) + +- Added the :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting that can be set + to ``True`` to make Scrapy abort HTTPS requests when the server certificate + is invalid or doesn't match the domain. + (:issue:`7379`) + +- The built-in HTTP :ref:`download handlers ` now + raise Scrapy-specific exceptions instead of implementation-specific ones, + to allow unified handling of similar problems caused by different + implementations. The default value of the :setting:`RETRY_EXCEPTIONS` + setting was updated replacing Twisted-specific exceptions with these new + ones. The exceptions: + + - :exc:`~scrapy.exceptions.CannotResolveHostError` + + - :exc:`~scrapy.exceptions.DownloadCancelledError` + + - :exc:`~scrapy.exceptions.DownloadConnectionRefusedError` + + - :exc:`~scrapy.exceptions.DownloadFailedError` + + - :exc:`~scrapy.exceptions.DownloadTimeoutError` + + - :exc:`~scrapy.exceptions.ResponseDataLossError` + + - :exc:`~scrapy.exceptions.UnsupportedURLSchemeError` + + (:issue:`7208`) + +- Added the :signal:`memusage_warning_reached` signal emitted by the + :class:`~scrapy.extensions.memusage.MemoryUsage` extension when the memory + usage reaches :setting:`MEMUSAGE_WARNING_MB`. + (:issue:`7249`, :issue:`7263`) + - Added :meth:`Headers.to_tuple_list() ` that returns headers as a list of ``(key, value)`` tuples. (:issue:`7239`) +- :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` now uses the + download handler configured for the ``"https"`` scheme to make requests + instead of always using + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + (:issue:`7369`, :issue:`7370`) + +- Added :func:`scrapy.utils.misc.walk_modules_iter` as a replacement for + ``scrapy.utils.misc.walk_modules()`` that returns an iterable instead of a + list. + (:issue:`7388`) + +Improvements +~~~~~~~~~~~~ + +- :func:`asyncio.to_thread` is now used instead of + :func:`twisted.internet.threads.deferToThread` in the built-in feed + storages, media pipeline storages and the + :func:`scrapy.utils.decorators.inthread` decorator when available. + (:issue:`7183`, :issue:`7184`, :issue:`7349`) + +- Improved memory footprint of :class:`~scrapy.Request` and + :class:`~scrapy.http.Response` objects by adding ``__slots__`` and omitting + empty lists and dicts in some internal attributes. + (:issue:`7036`, :issue:`7367`, :issue:`7374`) + +- :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory` + no longer mutates the SSL context, to avoid the behavior that was + deprecated in pyOpenSSL 25.1.0. + (:issue:`6859`, :issue:`7353`) + +- Improved memory usage of :class:`~scrapy.spiders.sitemap.SitemapSpider` and + :class:`scrapy.utils.sitemap.Sitemap`. + (:issue:`3529`, :issue:`7007`) + +- Improved the scheduling behavior of + :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` when crawling + multiple domains. + (:issue:`7293`, :issue:`7351`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now handle + TLS verbose logging (see :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING`) + directly instead of relying on + :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory`. + (:issue:`7387`) + +- The server certificate verification code now correctly handles certificates + with IP addresses in ``subjectAltName``. + (:issue:`7353`) + +- Improved reliability of :func:`scrapy.utils.trackref.get_oldest`. + (:issue:`1758`, :issue:`7375`) + +- Other code refactoring and improvements. + (:issue:`7210`, :issue:`7238`, :issue:`7376`, :issue:`7386`, :issue:`7395`, + :issue:`7405`, :issue:`7410`) + +Bug fixes +~~~~~~~~~ + +- :ref:`Media pipelines ` should now wait for uploads + to asynchronous storages (e.g. + :class:`~scrapy.pipelines.files.S3FilesStore`) to complete. + (:issue:`2183`, :issue:`6369`, :issue:`7182`) + +- Fixed merging ``*_BASE`` settings (e.g. merging + :setting:`DOWNLOADER_MIDDLEWARES` with + :setting:`DOWNLOADER_MIDDLEWARES_BASE`) when a component is referred to by + a class object in one setting and by a string import path in the other one. + (:issue:`6912`, :issue:`6993`) + +- ``scrapy runspider`` and ``scrapy crawl`` now set the exit code to 1 if an + exception happened early (this was broken since Scrapy 2.13.0). + (:issue:`6820`, :issue:`7255`) + +- Fixed repeated warnings about data loss (see + :setting:`DOWNLOAD_FAIL_ON_DATALOSS`) not being suppressed in + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + (:issue:`7222`) + +- Improved FTP connection management in + :class:`scrapy.pipelines.files.FTPFilesStore`. + (:issue:`7256`) + +- Fixed the ``spider`` variable in the :ref:`shell `, which + wasn't available since Scrapy 2.13.0. + (:issue:`7395`) + +Documentation +~~~~~~~~~~~~~ + +- The ``llms.txt`` and ``llms-full.txt`` files and Markdown versions of pages + are now generated when the HTML documentation is built. + (:issue:`7380`) + +- Added a "Copy as Markdown" button to the HTML documentation. + (:issue:`7380`) + +- Added :ref:`docs for using Pydantic models as items `. + (:issue:`6955`, :issue:`6966`) + +- Documented :ref:`job directory contents `. + (:issue:`4842`, :issue:`5260`) + +- Improved docs for :attr:`~scrapy.Request.dont_filter`. + (:issue:`6398`, :issue:`7245`) + +- Clarified that settings related to :setting:`TWISTED_DNS_RESOLVER` are only + taken into account if the selected resolver supports them. + (:issue:`7385`) + +- Other documentation improvements and fixes. + (:issue:`7248`, :issue:`7274`, :issue:`7406`, :issue:`7408`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added the ``no-reactor`` test environment that doesn't install a Twisted + reactor and uses ``pytest-asyncio`` instead of ``pytest-twisted`` to run + asynchronous test functions. + (:issue:`6952`, :issue:`7189`, :issue:`7233`, :issue:`7234`, :issue:`7254`, + :issue:`7259`) + +- Fixed running tests with ``pytest-xdist``. + (:issue:`7216`, :issue:`7257`) + +- Type hints improvements and fixes. + (:issue:`7300`, :issue:`7331`) + +- CI and test improvements and fixes. + (:issue:`7060`, + :issue:`7223`, + :issue:`7232`, + :issue:`7241`, + :issue:`7250`, + :issue:`7256`, + :issue:`7276`, + :issue:`7277`, + :issue:`7279`, + :issue:`7329`, + :issue:`7363`, + :issue:`7381`, + :issue:`7402`) + +.. _release-2.14.2: + +Scrapy 2.14.2 (2026-03-12) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Values from the ``Referrer-Policy`` header of HTTP responses are no longer + executed as Python callables. See the `cwxj-rr6w-m6w7`_ security advisory + for details. + + .. _cwxj-rr6w-m6w7: https://github.com/scrapy/scrapy/security/advisories/GHSA-cwxj-rr6w-m6w7 + +- In line with the `standard + `__, 301 redirects of + ``POST`` requests are converted into ``GET`` requests. + + Converting to a ``GET`` request implies not only a method change, but also + omitting the body and ``Content-*`` headers in the redirect request. On + cross-origin redirects (for example, cross-domain redirects), this is + effectively a security bug fix for scenarios where the body contains + secrets. + +Deprecations +~~~~~~~~~~~~ + +- Passing a response URL string as the first positional argument to + :meth:`scrapy.spidermiddlewares.referer.RefererMiddleware.policy` is + deprecated. Pass a :class:`~scrapy.http.Response` instead. + + The parameter has also been renamed to ``response`` to reflect this change. + The old parameter name (``resp_or_url``) is deprecated. + +New features +~~~~~~~~~~~~ + +- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing + supported referrer policies. + +Bug fixes +~~~~~~~~~ + +- Made additional redirect scenarios convert to ``GET`` in line with the + `standard `__: + + - Only ``POST`` 302 redirects are converted into ``GET`` requests; other + methods are preserved. + + - ``HEAD`` 303 redirects are not converted into ``GET`` requests. + + - ``GET`` 303 redirects do not have their body or standard ``Content-*`` + headers removed. + +- Redirects where the original request body is dropped now also have their + ``Content-Encoding``, ``Content-Language`` and ``Content-Location`` headers + removed, in addition to the ``Content-Type`` and ``Content-Length`` headers + that were already being removed. + +- Redirects now preserve the source URL fragment if the redirect URL does not + include one. This is useful when using browser-based download handlers, + such as `scrapy-playwright`_ or `scrapy-zyte-api`_, while letting Scrapy + handle redirects. + + .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright + .. _scrapy-zyte-api: https://scrapy-zyte-api.readthedocs.io/en/latest/ + +- The ``Referer`` header is now removed on redirect if + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` is disabled. + +- The handling of the ``Referer`` header on redirects now takes into account + the ``Referer-Policy`` header of the response that triggers the redirect. + .. _release-2.14.1: Scrapy 2.14.1 (2026-01-12) @@ -572,7 +1305,7 @@ New features (:issue:`4463`, :issue:`6804`) - Added :func:`scrapy.utils.asyncio.is_asyncio_available` as an alternative - to :func:`scrapy.utils.defer.is_asyncio_reactor_installed` with a + to :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` with a future-proof name and semantics. (:issue:`6827`) @@ -900,8 +1633,8 @@ Highlights: - Added the :reqmeta:`allow_offsite` request meta key -- :ref:`Spider middlewares that don't support asynchronous spider output - ` are deprecated +- Spider middlewares that don't support asynchronous spider output are + deprecated - Added a base class for :ref:`universal spider middlewares ` @@ -1039,13 +1772,11 @@ Deprecations ``start_queue_cls`` parameter. (:issue:`6752`) -- :ref:`Spider middlewares that don't support asynchronous spider output - ` are deprecated. The async iterable - downgrading feature, needed for using such middlewares with asynchronous - callbacks and with other spider middlewares that produce asynchronous - iterables, is also deprecated. Please update all such middlewares to - support asynchronous spider output. - (:issue:`6664`) +- Spider middlewares that don't support asynchronous spider output are + deprecated. The async iterable downgrading feature, needed for using such + middlewares with asynchronous callbacks and with other spider middlewares + that produce asynchronous iterables, is also deprecated. Please update all + such middlewares to support asynchronous spider output. (:issue:`6664`) - Functions that were imported from :mod:`w3lib.url` and re-exported in :mod:`scrapy.utils.url` are now deprecated, you should import them from @@ -1304,9 +2035,8 @@ Documentation - Documented the setting values set in the default project template. (:issue:`6762`, :issue:`6775`) -- Improved the :ref:`docs ` about asynchronous - iterable support in spider middlewares. - (:issue:`6688`) +- Improved the docs about asynchronous iterable support in spider + middlewares. (:issue:`6688`) - Improved the :ref:`docs ` about using :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based @@ -1470,7 +2200,7 @@ Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - User-defined cookies for HTTPS requests will have the ``secure`` flag set - to ``True`` unless it's set to ``False`` explictly. This is important when + to ``True`` unless it's set to ``False`` explicitly. This is important when these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP URL. (:issue:`6357`) @@ -1505,7 +2235,7 @@ Backward-incompatible changes ``crawler.settings`` instead. When they call ``__init__()`` of the base class they should pass the ``crawler`` argument to it too. - A ``from_settings()`` method shouldn't be defined. Class-specific - initialization code should go into either an overriden ``from_crawler()`` + initialization code should go into either an overridden ``from_crawler()`` method or into ``__init__()``. - It's now possible to override ``from_crawler()`` and it's not necessary to call ``MediaPipeline.from_crawler()`` in it if other recommendations @@ -4640,7 +5370,7 @@ Highlights: * :ref:`FTP support ` for media pipelines * New :attr:`Response.certificate ` attribute -* IPv6 support through :setting:`DNS_RESOLVER` +* IPv6 support through ``DNS_RESOLVER`` Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -4748,7 +5478,7 @@ New features :class:`twisted.internet.ssl.Certificate` object for HTTPS responses (:issue:`2726`, :issue:`4054`) -* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support +* A new ``DNS_RESOLVER`` setting allows enabling IPv6 support (:issue:`1031`, :issue:`4227`) * A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring @@ -4828,7 +5558,7 @@ New features components already supported (:issue:`4126`) * :class:`scrapy.utils.python.MutableChain.__iter__` now returns ``self``, - `allowing it to be used as a sequence `_ + allowing it to be used as a sequence. (:issue:`4153`) @@ -5278,7 +6008,7 @@ Backward-incompatible changes consistency with similar classes (:issue:`3929`, :issue:`3982`) * If you are using a custom context factory - (:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must + (``DOWNLOADER_CLIENTCONTEXTFACTORY``), its ``__init__`` method must accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers`` (:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`) @@ -6734,7 +7464,7 @@ This 1.1 release brings a lot of interesting features and bug fixes: selectors engine without needing to upgrade Scrapy. - HTTPS downloader now does TLS protocol negotiation by default, instead of forcing TLS 1.0. You can also set the SSL/TLS method - using the new :setting:`DOWNLOADER_CLIENT_TLS_METHOD`. + using the new ``DOWNLOADER_CLIENT_TLS_METHOD`` setting. - These bug fixes may require your attention: @@ -6769,8 +7499,7 @@ Keep reading for more details on other improvements and bug fixes. Beta Python 3 Support ~~~~~~~~~~~~~~~~~~~~~ -We have been `hard at work to make Scrapy run on Python 3 -`_. As a result, now +We have been hard at work to make Scrapy run on Python 3. As a result, now you can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some features are still missing (and some may never be ported). @@ -6838,7 +7567,7 @@ Additional New Features and Enhancements - Other refactoring, optimizations and cleanup (:issue:`1476`, :issue:`1481`, :issue:`1477`, :issue:`1315`, :issue:`1290`, :issue:`1750`, :issue:`1881`). -.. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md +.. _Code of Conduct: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md Deprecations and Removals diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 000000000..a1f3a7468 --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,8 @@ +h2 +pydantic +scrapy-spider-metadata +sphinx +sphinx-notfound-page +sphinx-rtd-theme +sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 diff --git a/docs/requirements.txt b/docs/requirements.txt index 500db7a7e..a5cbad302 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,197 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile -p 3.13 docs/requirements.in -o docs/requirements.txt +alabaster==1.0.0 + # via sphinx +annotated-types==0.7.0 + # via pydantic +attrs==26.1.0 + # via + # service-identity + # twisted +automat==25.4.16 + # via twisted +babel==2.18.0 + # via sphinx +certifi==2026.2.25 + # via requests +cffi==2.0.0 + # via cryptography +charset-normalizer==3.4.6 + # via requests +constantly==23.10.4 + # via twisted +cryptography==46.0.6 + # via + # pyopenssl + # scrapy + # service-identity +cssselect==1.4.0 + # via + # parsel + # scrapy +defusedxml==0.7.1 + # via scrapy +docutils==0.22.4 + # via + # sphinx + # sphinx-markdown-builder + # sphinx-rtd-theme +filelock==3.25.2 + # via tldextract h2==4.3.0 -pydantic==2.12.3 + # via -r docs/requirements.in +hpack==4.1.0 + # via h2 +hyperframe==6.1.0 + # via h2 +hyperlink==21.0.0 + # via twisted +idna==3.11 + # via + # hyperlink + # requests + # tldextract +imagesize==2.0.0 + # via sphinx +incremental==24.11.0 + # via twisted +itemadapter==0.13.1 + # via + # itemloaders + # scrapy +itemloaders==1.4.0 + # via scrapy +jinja2==3.1.6 + # via sphinx +jmespath==1.1.0 + # via + # itemloaders + # parsel +lxml==6.0.2 + # via + # parsel + # scrapy +markupsafe==3.0.3 + # via jinja2 +packaging==26.0 + # via + # incremental + # parsel + # scrapy + # scrapy-spider-metadata + # sphinx + # sphinx-scrapy +parsel==1.11.0 + # via + # itemloaders + # scrapy +protego==0.6.0 + # via scrapy +pyasn1==0.6.3 + # via + # pyasn1-modules + # service-identity +pyasn1-modules==0.4.2 + # via service-identity +pycparser==3.0 + # via cffi +pydantic==2.12.5 + # via + # -r docs/requirements.in + # scrapy-spider-metadata +pydantic-core==2.41.5 + # via pydantic +pydispatcher==2.0.7 + # via scrapy +pygments==2.19.2 + # via sphinx +pyopenssl==26.0.0 + # via scrapy +queuelib==1.9.0 + # via scrapy +requests==2.33.0 + # via + # requests-file + # sphinx + # tldextract +requests-file==3.0.1 + # via tldextract +roman-numerals==4.1.0 + # via sphinx +scrapy==2.14.2 + # via scrapy-spider-metadata scrapy-spider-metadata==0.2.0 -sphinx==8.1.3 -sphinx-notfound-page==1.0.4 -sphinx-rtd-theme==3.0.2 + # via -r docs/requirements.in +service-identity==24.2.0 + # via scrapy +snowballstemmer==3.0.1 + # via sphinx +sphinx==9.1.0 + # via + # -r docs/requirements.in + # sphinx-copybutton + # sphinx-last-updated-by-git + # sphinx-llms-txt + # sphinx-markdown-builder + # sphinx-notfound-page + # sphinx-rtd-theme + # sphinx-scrapy + # sphinxcontrib-jquery +sphinx-copybutton==0.5.2 + # via sphinx-scrapy +sphinx-last-updated-by-git==0.3.8 + # via sphinx-sitemap +sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69 + # via sphinx-scrapy +sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@cfe4c0bfd7b4542f7e6b65a58cdf9ec765829940 + # via sphinx-scrapy +sphinx-notfound-page==1.1.0 + # via -r docs/requirements.in sphinx-rtd-dark-mode==1.3.0 + # via -r docs/requirements.in +sphinx-rtd-theme==3.1.0 + # via + # -r docs/requirements.in + # sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 + # via -r docs/requirements.in +sphinx-sitemap==2.9.0 + # via sphinx-scrapy +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +tabulate==0.10.0 + # via sphinx-markdown-builder +tldextract==5.3.1 + # via scrapy +twisted==25.5.0 + # via scrapy +typing-extensions==4.15.0 + # via + # pydantic + # pydantic-core + # twisted + # typing-inspection +typing-inspection==0.4.2 + # via pydantic +urllib3==2.6.3 + # via requests +w3lib==2.4.1 + # via + # parsel + # scrapy +zope-interface==8.2 + # via + # scrapy + # twisted diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 64f00f622..01e4bcac1 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -98,9 +98,9 @@ recommend that such custom components should be written in the following way: (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default setting to the component provided by the add-on (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, - they shouldn't change it. + it should not be changed. 3. This way, if there are several add-ons that want to modify the same setting, - all of them will fallback to the component from the previous one and then to + all of them will fall back to the component from the previous one and then to the Scrapy default. The order of that depends on the priority order in the ``ADDONS`` setting. @@ -166,7 +166,7 @@ Use a fallback component: .. code-block:: python - from scrapy.utils.misc import build_from_crawler + from scrapy.utils.misc import build_from_crawler, load_object FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index e8c510ea5..c60c43f3c 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -63,7 +63,7 @@ this: :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`). 8. The :ref:`Engine ` sends processed items to - :ref:`Item Pipelines `, then send processed Requests to + :ref:`Item Pipelines `, then sends processed Requests to the :ref:`Scheduler ` and asks for possible next Requests to crawl. diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index b2b17a408..1a151044f 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -4,19 +4,27 @@ asyncio ======= -Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the -asyncio reactor `, you may use :mod:`asyncio` and -:mod:`asyncio`-powered libraries in any :doc:`coroutine `. +Scrapy supports :mod:`asyncio` natively. New projects created with +:command:`scrapy startproject` have asyncio enabled by default, and you can use +:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine +`. + +The rest of this page covers advanced topics. If you are starting a new project, +no additional setup is needed. .. _install-asyncio: -Installing the asyncio reactor -============================== +Configuring the asyncio reactor +=============================== -To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs -to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, -which is the default value. +New projects generated with :command:`scrapy startproject` have the asyncio +reactor configured by default. No manual setup is needed. + +The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy +uses. Its default value is +``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which enables +:mod:`asyncio` support. If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or :class:`~scrapy.crawler.CrawlerRunner`, you also need to @@ -129,6 +137,173 @@ example: .. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed +.. _asyncio-without-reactor: + +Using Scrapy without a Twisted reactor +====================================== + +.. versionadded:: 2.15.0 + +.. warning:: + This is currently experimental and may not be suitable for production use. + +It's possible to use Scrapy without installing a Twisted reactor at all, by +setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this +mode Scrapy will use the asyncio event loop directly, and most of the Scrapy +functionality will work in the same way. + +Doing this provides several benefits in certain use cases: + +* A Twisted reactor, once stopped, cannot be started again. This prevents, for + example, using several instances of + :class:`~scrapy.crawler.AsyncCrawlerProcess` in the same process when they + use a reactor, but with ``TWISTED_REACTOR_ENABLED=False`` it becomes + possible. +* There may be limitations imposed by + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` and related + Twisted code, such as the requirement of using + :class:`~asyncio.SelectorEventLoop` on Windows (see :ref:`asyncio-windows`), + that do not apply if the reactor is not used. +* :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` manages the + underlying event loop, and while :class:`~scrapy.crawler.AsyncCrawlerRunner` + can use a pre-existing reactor which, in turn, can use a pre-existing event + loop, it's easier to use :class:`~scrapy.crawler.AsyncCrawlerRunner` with a + pre-existing loop directly. +* Omitting the reactor machinery may improve performance and reliability. + +Limitations +----------- + +As some Scrapy features and components require a reactor, they don't work and +are disabled without it. Replacements that don't require a reactor may be added +in future Scrapy versions. The following features are not available: + +* The default HTTP(S) download handler, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` (this + is likely the biggest difference; Scrapy provides an HTTP(S) download handler + that doesn't require a reactor and will be used instead of it: + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`) +* :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` +* :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` +* :ref:`topics-telnetconsole` +* :class:`~scrapy.crawler.CrawlerRunner` and + :class:`~scrapy.crawler.CrawlerProcess` + (:class:`~scrapy.crawler.AsyncCrawlerProcess` and + :class:`~scrapy.crawler.AsyncCrawlerRunner` are available) +* Twisted-specific DNS resolvers (the :setting:`DNS_RESOLVER` setting) +* User and 3rd-party code that requires a reactor (see :ref:`below + ` for examples) + +Note that importing Twisted modules and, among other things, creating and using +:class:`~twisted.internet.defer.Deferred` objects doesn't require a reactor, so +code that uses :class:`~twisted.internet.defer.Deferred`, +:class:`~twisted.python.failure.Failure` and some other Twisted APIs will not +necessarily stop working. + +Other differences +----------------- + +When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Scrapy will change +the defaults of some other settings: + +* :setting:`TELNETCONSOLE_ENABLED` is set to ``False``. +* The ``"http"`` and ``"https"`` keys in :setting:`DOWNLOAD_HANDLERS_BASE` are + set to ``"scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler"``. +* The ``"ftp"`` key in :setting:`DOWNLOAD_HANDLERS_BASE` is set to ``None``. + +Thus, :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` is +used by default for making HTTP(S) requests. Please refer to its documentation +for its differences and limitations compared to +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + +Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a +:term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from +being imported. + +.. _asyncio-without-reactor-migrate: + +Adding support to existing code +------------------------------- + +Code that doesn't directly use Twisted APIs or APIs that depend on Twisted ones +doesn't need special support for running without a reactor. + +Here are some examples of APIs and patterns that need a replacement: + +* Using :meth:`reactor.callLater() + ` for sleeping or delayed calls. + You can use :meth:`asyncio.loop.call_later` instead. +* Using :func:`twisted.internet.threads.deferToThread`, + :meth:`reactor.callFromThread() + ` and related APIs to + execute code in other threads. You can use :func:`asyncio.to_thread`, + :meth:`asyncio.loop.call_soon_threadsafe` and related APIs instead. +* Using :class:`twisted.internet.task.LoopingCall` for scheduling repeated + tasks. As there is no direct replacement in the standard library, you may + need to write your own one using :func:`asyncio.sleep` in a task. +* Using Twisted network client and server APIs (:meth:`reactor.connectTCP() + `, + :meth:`reactor.listenTCP() + `, + :mod:`twisted.web.client`, :mod:`twisted.mail.smtp` etc.). You can use other + built-in or 3rd-party libraries for this. +* Using :class:`~scrapy.crawler.CrawlerProcess` or + :class:`~scrapy.crawler.CrawlerRunner`. You should use + :class:`~scrapy.crawler.AsyncCrawlerProcess` or + :class:`~scrapy.crawler.AsyncCrawlerRunner` respectively instead. +* Checking whether ``asyncio`` support is available with + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`. You should use + :func:`scrapy.utils.asyncio.is_asyncio_available` instead. + +Scrapy provides unified helpers for some of these examples: + +.. autofunction:: scrapy.utils.asyncio.call_later +.. autofunction:: scrapy.utils.asyncio.create_looping_call +.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall +.. autofunction:: scrapy.utils.asyncio.run_in_thread + +If your code needs to know whether the reactor is available, you can either +check for the value of the :setting:`TWISTED_REACTOR_ENABLED` setting (you need +access to the :class:`~scrapy.crawler.Crawler` instance to do this) or use the +following function: + +.. autofunction:: scrapy.utils.reactorless.is_reactorless + +In general, code that doesn't use the reactor (directly or indirectly) can be +used unmodified both with the asyncio reactor and without a reactor. This +includes code that converts Deferreds to futures and vice versa as described in +:ref:`asyncio-await-dfd`. + +Troubleshooting +--------------- + +**ImportError: Import of twisted.internet.reactor is forbidden when running +without a Twisted reactor [...]:** Scrapy is configured to run without a +reactor, but some code imported :mod:`twisted.internet.reactor`, most likely +because that code needs a reactor to be used. You need to stop using this code +or set :setting:`TWISTED_REACTOR_ENABLED` back to ``True``. It's also possible +that the reactor isn't really needed but was installed due to the problem +described in :ref:`asyncio-preinstalled-reactor`, in which case it should be +enough to fix the problematic imports. + +**RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is +installed:** Scrapy is configured to run without a reactor, but a reactor is +already installed before the Scrapy code is executed. If you are trying to set +:setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. + +**RuntimeError: We expected a Twisted reactor to be installed but it isn't:** +Scrapy is configured to run with a reactor and not to install one, but a +reactor wasn't installed before the Scrapy code is executed. If you are trying +to set :setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. + +**RuntimeError: doesn't support TWISTED_REACTOR_ENABLED=False:** The +listed class cannot be used with :setting:`TWISTED_REACTOR_ENABLED` set to +``False``. There may be a replacement in the :ref:`documentation above +` or the documentation of the affected class. + + .. _asyncio-windows: Windows-specific notes @@ -149,6 +324,9 @@ automatically when you change the :setting:`TWISTED_REACTOR` setting or call them together with Scrapy on Windows (but you should be able to use them on WSL or native Linux). +.. note:: This problem doesn't apply when not using the reactor, see + :ref:`asyncio-without-reactor`. + .. _playwright: https://github.com/microsoft/playwright-python diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 56f8c6498..c0df86922 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -11,12 +11,10 @@ That includes the classes that you may assign to the following settings: - :setting:`ADDONS` -- :setting:`DNS_RESOLVER` +- :setting:`TWISTED_DNS_RESOLVER` - :setting:`DOWNLOAD_HANDLERS` -- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` - - :setting:`DOWNLOADER_MIDDLEWARES` - :setting:`DUPEFILTER_CLASS` diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a2dd67f68..116aac323 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -23,9 +23,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.Request` callbacks. - If you are using any custom or third-party :ref:`spider middleware - `, see :ref:`sync-async-spider-middleware`. - - The :meth:`process_item` method of :ref:`item pipelines `. @@ -39,13 +36,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` - method of :ref:`spider middlewares `. - - If defined as a coroutine, it must be an :term:`asynchronous generator`. - The input ``result`` parameter is an :term:`asynchronous iterable`. - - See also :ref:`sync-async-spider-middleware` and - :ref:`universal-spider-middleware`. + method of :ref:`spider middlewares `, which + *must* be defined as an :term:`asynchronous generator` except in + :ref:`universal spider middlewares `. - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method of :ref:`spider middlewares `, which *must* be @@ -73,12 +66,6 @@ In the future we plan to add support for the ``async def`` syntax to these APIs or replace them with other APIs where changing the existing ones isn't possible. -These APIs don't have a coroutine-based counterpart: - -- :class:`~scrapy.mail.MailSender` - - - :meth:`~scrapy.mail.MailSender.send` - These APIs have a coroutine-based implementation and a Deferred-based one: - :class:`scrapy.crawler.Crawler`: @@ -137,18 +124,11 @@ wrapping a :class:`~twisted.internet.defer.Deferred` object into a :class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for more information about this. -For example: - -- The :meth:`MailSender.send() ` method returns - a :class:`~twisted.internet.defer.Deferred` object that fires when the - email is sent. You can use this object directly in Deferred-based code or - convert it into a :class:`~asyncio.Future` object with - :func:`~scrapy.utils.defer.maybe_deferred_to_future`. -- A custom scheduler needs to define an ``open()`` method that can return a - :class:`~twisted.internet.defer.Deferred` object. You can write a method - that works with Deferreds and returns one directly, or you can write a - coroutine and convert it into a function that returns a Deferred with - :func:`~scrapy.utils.defer.deferred_f_from_coro_f`. +For example: a custom scheduler needs to define an ``open()`` method that can +return a :class:`~twisted.internet.defer.Deferred` object. You can write a +method that works with Deferreds and returns one directly, or you can write a +coroutine and convert it into a function that returns a Deferred with +:func:`~scrapy.utils.defer.deferred_f_from_coro_f`. General usage @@ -287,139 +267,6 @@ You can also send multiple requests in parallel: responses = await asyncio.gather(*tasks) yield { "h1": response.css("h1::text").get(), - "price": responses[0][1].css(".price::text").get(), - "price2": responses[1][1].css(".color::text").get(), + "price": responses[0].css(".price::text").get(), + "price2": responses[1].css(".color::text").get(), } - - -.. _sync-async-spider-middleware: - -Mixing synchronous and asynchronous spider middlewares -====================================================== - -The output of a :class:`~scrapy.Request` callback is passed as the ``result`` -parameter to the -:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method -of the first :ref:`spider middleware ` from the -:ref:`list of active spider middlewares `. -Then the output of that ``process_spider_output`` method is passed to the -``process_spider_output`` method of the next spider middleware, and so on for -every active spider middleware. - -Scrapy supports mixing :ref:`coroutine methods ` and synchronous methods -in this chain of calls. - -However, if any of the ``process_spider_output`` methods is defined as a -synchronous method, and the previous ``Request`` callback or -``process_spider_output`` method is a coroutine, there are some drawbacks to -the asynchronous-to-synchronous conversion that Scrapy does so that the -synchronous ``process_spider_output`` method gets a synchronous iterable as its -``result`` parameter: - -- The whole output of the previous ``Request`` callback or - ``process_spider_output`` method is awaited at this point. - -- If an exception raises while awaiting the output of the previous - ``Request`` callback or ``process_spider_output`` method, none of that - output will be processed. - - This contrasts with the regular behavior, where all items yielded before - an exception raises are processed. - -Asynchronous-to-synchronous conversions are supported for backward -compatibility, but they are deprecated and will stop working in a future -version of Scrapy. - -To avoid asynchronous-to-synchronous conversions, when defining ``Request`` -callbacks as coroutine methods or when using spider middlewares whose -``process_spider_output`` method is an :term:`asynchronous generator`, all -active spider middlewares must either have their ``process_spider_output`` -method defined as an asynchronous generator or :ref:`define a -process_spider_output_async method `. - -.. _sync-async-spider-middleware-users: - -For middleware users --------------------- - -If you have asynchronous callbacks or use asynchronous-only spider middlewares -you should make sure the asynchronous-to-synchronous conversions -:ref:`described above ` don't happen. To do this, -make sure all spider middlewares you use support asynchronous spider output. -Even if you don't have asynchronous callbacks and don't use asynchronous-only -spider middlewares in your project, it's still a good idea to make sure all -middlewares you use support asynchronous spider output, so that it will be easy -to start using asynchronous callbacks in the future. Because of this, Scrapy -logs a warning when it detects a synchronous-only spider middleware. - -If you want to update middlewares you wrote, see the :ref:`following section -`. If you have 3rd-party middlewares that -aren't yet updated by their authors, you can :ref:`subclass ` -them to make them :ref:`universal ` and use the -subclasses in your projects. - -.. _sync-async-spider-middleware-authors: - -For middleware authors ----------------------- - -If you have a spider middleware that defines a synchronous -``process_spider_output`` method, you should update it to support asynchronous -spider output for :ref:`better compatibility `, -even if you don't yet use it with asynchronous callbacks, especially if you -publish this middleware for other people to use. You have two options for this: - -1. Make the middleware asynchronous, by making the ``process_spider_output`` - method an :term:`asynchronous generator`. -2. Make the middleware universal, as described in the :ref:`next section - `. - -If your middleware won't be used in projects with synchronous-only middlewares, -e.g. because it's an internal middleware and you know that all other -middlewares in your projects are already updated, it's safe to choose the first -option. Otherwise, it's better to choose the second option. - -.. _universal-spider-middleware: - -Universal spider middlewares ----------------------------- - -To allow writing a spider middleware that supports asynchronous execution of -its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding -:ref:`asynchronous-to-synchronous conversions `) -while maintaining support for older Scrapy versions, you may define -``process_spider_output`` as a synchronous method and define an -:term:`asynchronous generator` version of that method with an alternative name: -``process_spider_output_async``. - -For example: - -.. code-block:: python - - class UniversalSpiderMiddleware: - def process_spider_output(self, response, result): - for r in result: - # ... do something with r - yield r - - async def process_spider_output_async(self, response, result): - async for r in result: - # ... do something with r - yield r - -.. note:: This is an interim measure to allow, for a time, to write code that - works in Scrapy 2.7 and later without requiring - asynchronous-to-synchronous conversions, and works in earlier Scrapy - versions as well. - - In some future version of Scrapy, however, this feature will be - deprecated and, eventually, in a later version of Scrapy, this - feature will be removed, and all spider middlewares will be expected - to define their ``process_spider_output`` method as an asynchronous - generator. - -Since 2.13.0, Scrapy provides a base class, -:class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which implements -the ``process_spider_output()`` and ``process_spider_output_async()`` methods, -so instead of duplicating the processing code you can override the -``get_processed_request()`` and/or the ``get_processed_item()`` method. diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 7e672f0b4..888bfaf08 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -102,43 +102,67 @@ An optional base class for custom handlers is provided: :undoc-members: :member-order: bysource +.. _download-handlers-exceptions: + +Exceptions raised by download handlers +====================================== + +.. versionadded:: 2.15.0 + +The built-in download handlers raise Scrapy-specific exceptions instead of +implementation-specific ones, so that code that handles these exceptions can be +written in a generic way. We recommend custom download handlers to also use +these exceptions. + +.. autoexception:: scrapy.exceptions.CannotResolveHostError + +.. autoexception:: scrapy.exceptions.DownloadCancelledError + +.. autoexception:: scrapy.exceptions.DownloadConnectionRefusedError + +.. autoexception:: scrapy.exceptions.DownloadFailedError + +.. autoexception:: scrapy.exceptions.DownloadTimeoutError + +.. autoexception:: scrapy.exceptions.ResponseDataLossError + +.. autoexception:: scrapy.exceptions.UnsupportedURLSchemeError + .. _download-handlers-ref: -Built-in download handlers reference -==================================== +Built-in HTTP download handlers reference +========================================= -DataURIDownloadHandler ----------------------- +Scrapy ships several handlers for HTTP and HTTPS requests. While all of them +support basic features, they may differ in support of specific Scrapy features +and settings and HTTP protocol features. See the documentation of specific +handlers and specific settings for more information. Additionally, as the +underlying HTTP client implementations differ between handlers, the behavior of +specific websites may be different when doing the same Scrapy requests but +using different handlers. -.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler +Here is a comparison of some features of the built-in HTTP handlers, see the +individual handler docs for more differences: -| Supported scheme: ``data``. -| Lazy: no. +================== ================= ===================== ==================== +Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler +================== ================= ===================== ==================== +Requires asyncio No No Yes +Requires a reactor Yes Yes No +HTTP/1.1 No Yes Yes +HTTP/2 Yes No Yes +TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` +HTTP proxies No Yes Yes +SOCKS proxies No No Yes +================== ================= ===================== ==================== -This handler supports RFC 2397 ``data:content/type;base64,`` data URIs. +You can find additional HTTP download handlers in the +scrapy-download-handlers-incubator_ package. This package is made by the Scrapy +developers and contains experimental handlers that may be included in some +later Scrapy version but can already be used. Please refer to the documentation +of this package for more information. -FileDownloadHandler -------------------- - -.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler - -| Supported scheme: ``file``. -| Lazy: no. - -This handler supports ``file:///path`` local file URIs. It doesn't -support remote files. - -FTPDownloadHandler ------------------- - -.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler - -| Supported scheme: ``ftp``. -| Lazy: no. - -This handler supports ``ftp://host/path`` FTP URIs. - -It's implemented using :mod:`twisted.protocols.ftp`. +.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator .. _twisted-http2-handler: @@ -148,7 +172,9 @@ H2DownloadHandler .. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler | Supported scheme: ``https``. -| Lazy: yes. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol for them. @@ -167,27 +193,43 @@ If you want to use this handler you need to replace the default one for the "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", } +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + .. warning:: This handler is experimental, and not yet recommended for production environments. Future Scrapy versions may introduce related changes without a deprecation period or warning. -.. note:: +=========================== ================================================ +HTTP proxies No (not implemented) +SOCKS proxies No (not supported by the library) +HTTP/2 Yes +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ - Known limitations of the HTTP/2 implementation in this handler include: +Other limitations: - - No support for HTTP/2 Cleartext (h2c), since no major browser supports - HTTP/2 unencrypted (refer `http2 faq`_). +- No support for HTTP/1.1. - - No setting to specify a maximum `frame size`_ larger than the default - value, 16384. Connections to servers that send a larger frame will - fail. +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. - - No support for `server pushes`_, which are ignored. +- No support for the :signal:`bytes_received` and :signal:`headers_received` + signals. - - No support for the :signal:`bytes_received` and - :signal:`headers_received` signals. +Known limitations of the HTTP/2 support: + +- No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). + +- No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will fail. + +- No support for `server pushes`_, which are ignored. .. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption @@ -199,20 +241,148 @@ HTTP11DownloadHandler .. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler | Supported schemes: ``http``, ``https``. -| Lazy: no. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. This handler supports ``http://host/path`` and ``https://host/path`` URLs and uses the HTTP/1.1 protocol for them. It's implemented using :mod:`twisted.web.client`. +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +=========================== ================================================ +HTTP proxies Yes +SOCKS proxies No (not supported by the library) +HTTP/2 No (implemented as a separate handler) +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- HTTPS proxies to HTTPS destinations are not supported. + +HttpxDownloadHandler +-------------------- + +.. versionadded:: 2.15.0 + +.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler + +| Supported schemes: ``http``, ``https``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: yes. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``http://host/path`` and ``https://host/path`` URLs and +uses the HTTP/1.1 or HTTP/2 protocol for them. + +It's implemented using the ``httpx`` library and needs it to be installed. + +If you want to use this handler you need to replace the default ones for the +``http`` and ``https`` schemes: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + } + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: + + This handler is experimental, and not yet recommended for production + environments. Future Scrapy versions may introduce related changes without + a deprecation period or warning or even remove it altogether. + +=========================== ======================================= +HTTP proxies Yes +SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``) +HTTP/2 Yes (requires ``httpx[http2]``) +``response.certificate`` DER bytes +Per-request ``bindaddress`` No (not supported by the library) +TLS implementation Standard library ``ssl`` +=========================== ======================================= + +Other limitations: + +- The handler creates a separate connection pool for each proxy URL (due to + limitations of ``httpx``) which may lead to higher resource usage when + using proxy rotation. + +.. setting:: HTTPX_HTTP2_ENABLED + +HTTPX_HTTP2_ENABLED +^^^^^^^^^^^^^^^^^^^ + +Default: ``False`` + +Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra +needs to be installed if you want to enable this setting. + +.. versionadded:: VERSION + +Built-in non-HTTP download handlers reference +============================================= + +DataURIDownloadHandler +---------------------- + +.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler + +| Supported scheme: ``data``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports RFC 2397 ``data:content/type;base64,`` data URIs. + +FileDownloadHandler +------------------- + +.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler + +| Supported scheme: ``file``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``file:///path`` local file URIs. It doesn't +support remote files. + +FTPDownloadHandler +------------------ + +.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler + +| Supported scheme: ``ftp``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``ftp://host/path`` FTP URIs. + +It's implemented using :mod:`twisted.protocols.ftp`. + S3DownloadHandler ----------------- .. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler | Supported scheme: ``s3``. -| Lazy: yes. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. This handler supports ``s3://bucket/path`` S3 URIs. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1a4f91d93..0c1af5276 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -307,26 +307,15 @@ HttpAuthMiddleware .. class:: HttpAuthMiddleware - This middleware authenticates all requests generated from certain spiders - using `Basic access authentication`_ (aka. HTTP auth). + This middleware authenticates requests using `Basic access authentication`_ + (aka. HTTP auth). - To enable HTTP authentication for a spider, set the ``http_user`` and - ``http_pass`` spider attributes to the authentication data and the - ``http_auth_domain`` spider attribute to the domain which requires this - authentication (its subdomains will be also handled in the same way). - You can set ``http_auth_domain`` to ``None`` to enable the - authentication for all requests but you risk leaking your authentication - credentials to unrelated domains. + Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and + :setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override + the credentials per request via :attr:`~scrapy.Request.meta` keys + :reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`. - .. warning:: - In previous Scrapy versions HttpAuthMiddleware sent the authentication - data with all requests, which is a security problem if the spider - makes requests to several different domains. Currently if the - ``http_auth_domain`` attribute is not set, the middleware will use the - domain of the first request, which will work for some spiders but not - for others. In the future the middleware will produce an error instead. - - Example: + Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`): .. code-block:: python @@ -334,13 +323,62 @@ HttpAuthMiddleware class SomeIntranetSiteSpider(CrawlSpider): - http_user = "someuser" - http_pass = "somepass" - http_auth_domain = "intranet.example.com" name = "intranet.example.com" + custom_settings = { + "HTTPAUTH_USER": "someuser", + "HTTPAUTH_PASS": "somepass", + "HTTPAUTH_DOMAIN": "intranet.example.com", + } # .. rest of the spider code omitted ... + Example using per-request meta: + + .. code-block:: python + + async def start(self): + yield Request( + "https://intranet.example.com/protected/", + meta={ + "http_user": "someuser", + "http_pass": "somepass", + "http_auth_domain": "intranet.example.com", + }, + ) + +.. setting:: HTTPAUTH_USER + +HTTPAUTH_USER +~~~~~~~~~~~~~ + +Default: ``""`` + +The username to use for HTTP basic authentication, applied to all requests +whose URL matches :setting:`HTTPAUTH_DOMAIN`. + +.. setting:: HTTPAUTH_PASS + +HTTPAUTH_PASS +~~~~~~~~~~~~~ + +Default: ``""`` + +The password to use for HTTP basic authentication. + +.. setting:: HTTPAUTH_DOMAIN + +HTTPAUTH_DOMAIN +~~~~~~~~~~~~~~~ + +Default: ``None`` + +The domain (and its subdomains) to which HTTP basic authentication credentials +are sent. Set to ``None`` to send credentials with all requests, but be aware +that this risks leaking credentials to unrelated domains. + +This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER` +or :setting:`HTTPAUTH_PASS` is set. + .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication @@ -726,7 +764,7 @@ HttpProxyMiddleware .. class:: HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the - ``proxy`` meta value for :class:`~scrapy.Request` objects. + :reqmeta:`proxy` meta value for :class:`~scrapy.Request` objects. Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: @@ -735,11 +773,35 @@ HttpProxyMiddleware * ``https_proxy`` * ``no_proxy`` - You can also set the meta key ``proxy`` per-request, to a value like + You can also set the meta key :reqmeta:`proxy` per-request, to a value like ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. +.. note:: + + Handling of this meta key needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. It's currently unsupported by + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`. + +.. note:: + + Usually a proxy URL uses the ``http://`` scheme. More rarely, it uses the + ``https://`` one. While both kinds of proxy URLs can be used with both HTTP + and HTTPS destination URLs, the specifics of the network exchange are + different for all 4 cases and it's possible that HTTPS proxies are fully or + partially unsupported by a given download handler. Currently, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + supports HTTPS proxies only for HTTP destinations. + +.. note:: + + If the download handler supports it, you can use a SOCKS proxy URL (e.g. + ``socks5://username:password@some_proxy_server:port``). + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` + supports SOCKS proxies while other built-in handlers don't. + HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1035,6 +1097,21 @@ has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught exception propagation, see :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. +.. setting:: RETRY_GIVE_UP_LOG_LEVEL + +RETRY_GIVE_UP_LOG_LEVEL +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"ERROR"`` + +:ref:`Logging level ` used for the message logged when a request +exceeds its retries. + +Can be a level name (e.g. ``"WARNING"``) or a number (e.g. ``logging.WARNING`` +or ``30``). + +See also: :reqmeta:`give_up_log_level`, :func:`get_retry_request`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST @@ -1096,7 +1173,7 @@ Parsers vary in several aspects: * Support for wildcard matching -* Usage of `length based rule `_: +* Usage of `length based rule `_: in particular for ``Allow`` and ``Disallow`` directives, where the most specific rule based on the length of the path trumps the less specific (shorter) rule @@ -1114,7 +1191,7 @@ Based on `Protego `_: * implemented in Python * is compliant with `Google's Robots.txt Specification - `_ + `_ * supports wildcard matching @@ -1134,9 +1211,9 @@ Based on :class:`~urllib.robotparser.RobotFileParser`: * is compliant with `Martijn Koster's 1996 draft specification `_ -* lacks support for wildcard matching +* lacks support for wildcard matching (before Python 3.14.5) -* doesn't use the length based rule +* doesn't use the length based rule (before Python 3.14.5) It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0. diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 18b3ce244..5a399e094 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -83,7 +83,7 @@ request with Scrapy. It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.FormRequest`) of that request. +form parameters (see :ref:`form`) of that request. As all major browsers allow to export the requests in curl_ format, Scrapy incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent @@ -274,16 +274,13 @@ However, using `playwright-python`_ directly as in the above example circumvents most of the Scrapy components (middlewares, dupefilter, etc). We recommend using `scrapy-playwright`_ for a better integration. -.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets -.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript .. _chompjs: https://github.com/Nykakin/chompjs .. _curl: https://curl.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser .. _js2xml: https://github.com/scrapinghub/js2xml .. _playwright-python: https://github.com/microsoft/playwright-python .. _playwright: https://github.com/microsoft/playwright -.. _pyppeteer: https://pyppeteer.github.io/pyppeteer/ .. _pytesseract: https://github.com/madmaze/pytesseract .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright .. _tabula-py: https://github.com/chezou/tabula-py diff --git a/docs/topics/email.rst b/docs/topics/email.rst deleted file mode 100644 index 1d7bad787..000000000 --- a/docs/topics/email.rst +++ /dev/null @@ -1,185 +0,0 @@ -.. _topics-email: - -============== -Sending e-mail -============== - -.. module:: scrapy.mail - :synopsis: Email sending facility - -Although Python makes sending e-mails relatively easy via the :mod:`smtplib` -library, Scrapy provides its own facility for sending e-mails which is very -easy to use and it's implemented using :doc:`Twisted non-blocking IO -`, to avoid interfering with the non-blocking -IO of the crawler. It also provides a simple API for sending attachments and -it's very easy to configure, with a few :ref:`settings -`. - -Quick example -============= - -There are two ways to instantiate the mail sender. You can instantiate it using -the standard ``__init__`` method: - -.. code-block:: python - - from scrapy.mail import MailSender - - mailer = MailSender() - -Or you can instantiate it passing a :class:`scrapy.Crawler` instance, which -will respect the :ref:`settings `: - -.. skip: start -.. code-block:: python - - mailer = MailSender.from_crawler(crawler) - -And here is how to use it to send an e-mail (without attachments): - -.. code-block:: python - - mailer.send( - to=["someone@example.com"], - subject="Some subject", - body="Some body", - cc=["another@example.com"], - ) -.. skip: end - -MailSender class reference -========================== - -The MailSender :ref:`components ` is the preferred class to -use for sending emails from Scrapy, as it uses :doc:`Twisted non-blocking IO -`, like the rest of the framework. - -.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) - - :param smtphost: the SMTP host to use for sending the emails. If omitted, the - :setting:`MAIL_HOST` setting will be used. - :type smtphost: str - - :param mailfrom: the address used to send emails (in the ``From:`` header). - If omitted, the :setting:`MAIL_FROM` setting will be used. - :type mailfrom: str - - :param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER` - setting will be used. If not given, no SMTP authentication will be - performed. - :type smtphost: str or bytes - - :param smtppass: the SMTP pass for authentication. - :type smtppass: str or bytes - - :param smtpport: the SMTP port to connect to - :type smtpport: int - - :param smtptls: enforce using SMTP STARTTLS - :type smtptls: bool - - :param smtpssl: enforce using a secure SSL connection - :type smtpssl: bool - - .. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None) - - Send email to the given recipients. - - :param to: the e-mail recipients as a string or as a list of strings - :type to: str or list - - :param subject: the subject of the e-mail - :type subject: str - - :param cc: the e-mails to CC as a string or as a list of strings - :type cc: str or list - - :param body: the e-mail body - :type body: str - - :param attachs: an iterable of tuples ``(attach_name, mimetype, - file_object)`` where ``attach_name`` is a string with the name that will - appear on the e-mail's attachment, ``mimetype`` is the mimetype of the - attachment and ``file_object`` is a readable file object with the - contents of the attachment - :type attachs: collections.abc.Iterable - - :param mimetype: the MIME type of the e-mail - :type mimetype: str - - :param charset: the character encoding to use for the e-mail contents - :type charset: str - - -.. _topics-email-settings: - -Mail settings -============= - -These settings define the default ``__init__`` method values of the :class:`MailSender` -class, and can be used to configure e-mail notifications in your project without -writing any code (for those extensions and code that uses :class:`MailSender`). - -.. setting:: MAIL_FROM - -MAIL_FROM ---------- - -Default: ``'scrapy@localhost'`` - -Sender email to use (``From:`` header) for sending emails. - -.. setting:: MAIL_HOST - -MAIL_HOST ---------- - -Default: ``'localhost'`` - -SMTP host to use for sending emails. - -.. setting:: MAIL_PORT - -MAIL_PORT ---------- - -Default: ``25`` - -SMTP port to use for sending emails. - -.. setting:: MAIL_USER - -MAIL_USER ---------- - -Default: ``None`` - -User to use for SMTP authentication. If disabled no SMTP authentication will be -performed. - -.. setting:: MAIL_PASS - -MAIL_PASS ---------- - -Default: ``None`` - -Password to use for SMTP authentication, along with :setting:`MAIL_USER`. - -.. setting:: MAIL_TLS - -MAIL_TLS --------- - -Default: ``False`` - -Enforce using STARTTLS. STARTTLS is a way to take an existing insecure connection, and upgrade it to a secure connection using SSL/TLS. - -.. setting:: MAIL_SSL - -MAIL_SSL --------- - -Default: ``False`` - -Enforce connecting using an SSL encrypted connection diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 74256eef4..c4cd05683 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -93,24 +93,25 @@ described next. 1. Declaring a serializer in the field -------------------------------------- -If you use :class:`~scrapy.Item` you can declare a serializer in the -:ref:`field metadata `. The serializer must be -a callable which receives a value and returns its serialized form. +Every :ref:`item type ` except :class:`dict` lets you declare a +serializer in the :ref:`field metadata `. The serializer +must be a callable which receives a value and returns its serialized form. Example: .. code-block:: python - import scrapy + from dataclasses import dataclass, field def serialize_price(value): return f"$ {str(value)}" - class Product(scrapy.Item): - name = scrapy.Field() - price = scrapy.Field(serializer=serialize_price) + @dataclass + class Product: + name: str + price: float = field(metadata={"serializer": serialize_price}) 2. Overriding the serialize_field() method diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 0fbd22a27..5a05f67d9 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -136,7 +136,18 @@ Core Stats extension Enable the collection of core statistics, provided the stats collection is enabled (see :ref:`topics-stats`). -.. _topics-extensions-ref-telnetconsole: +The following stats are collected: + +* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`). +* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`). +* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`). +* ``finish_reason``: the closing reason string (e.g. ``"finished"``, + ``"closespider_timeout"``). +* ``item_scraped_count``: total number of items that passed all pipelines. +* ``item_dropped_count``: total number of items dropped by a pipeline. +* ``item_dropped_reasons_count/``: per-exception drop count + (e.g. ``item_dropped_reasons_count/DropItem``). +* ``response_received_count``: total number of HTTP responses received. Log Count extension ~~~~~~~~~~~~~~~~~~~ @@ -146,6 +157,8 @@ Log Count extension .. autoclass:: LogCount +.. _topics-extensions-ref-telnetconsole: + Telnet console extension ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -175,20 +188,16 @@ Memory usage extension Monitors the memory used by the Scrapy process that runs the spider and: -1. sends a notification e-mail when it exceeds a certain value -2. closes the spider when it exceeds a certain value - -The notification e-mails can be triggered when a certain warning value is -reached (:setting:`MEMUSAGE_WARNING_MB`) and when the maximum value is reached -(:setting:`MEMUSAGE_LIMIT_MB`) which will also cause the spider to be closed -and the Scrapy process to be terminated. +1. sends a :signal:`memusage_warning_reached` signal when it exceeds + :setting:`MEMUSAGE_WARNING_MB` +2. closes the spider with the `"memusage_exceeded"` reason when it exceeds + :setting:`MEMUSAGE_LIMIT_MB` This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and can be configured with the following settings: * :setting:`MEMUSAGE_LIMIT_MB` * :setting:`MEMUSAGE_WARNING_MB` -* :setting:`MEMUSAGE_NOTIFY_MAIL` * :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` Memory debugger extension @@ -251,6 +260,7 @@ settings: * :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM` * :setting:`CLOSESPIDER_ITEMCOUNT` * :setting:`CLOSESPIDER_PAGECOUNT` +* :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` * :setting:`CLOSESPIDER_ERRORCOUNT` .. note:: @@ -264,12 +274,11 @@ settings: CLOSESPIDER_TIMEOUT """"""""""""""""""" -Default: ``0`` +Default: ``0.0`` -An integer which specifies a number of seconds. If the spider remains open for -more than that number of seconds, it will be automatically closed with the -reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by -timeout. +If the spider remains open for more than this number of seconds, it will be +automatically closed with the reason ``closespider_timeout``. If zero (or non +set), spiders won't be closed by timeout. .. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM @@ -332,24 +341,6 @@ closing the spider. If the spider generates more than that number of errors, it will be closed with the reason ``closespider_errorcount``. If zero (or non set), spiders won't be closed by number of errors. -StatsMailer extension -~~~~~~~~~~~~~~~~~~~~~ - -.. module:: scrapy.extensions.statsmailer - :synopsis: StatsMailer extension - -.. class:: StatsMailer - -This simple extension can be used to send a notification e-mail every time a -domain has finished scraping, including the Scrapy stats collected. The email -will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS` -setting. - -Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a -full list of parameters, including examples on how to instantiate -:class:`~scrapy.mail.MailSender` and use mail settings, see -:ref:`topics-email`. - .. module:: scrapy.extensions.debug :synopsis: Extensions for debugging Scrapy diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index a5537c39e..6ce5255b9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -246,7 +246,7 @@ The feeds are stored on `Google Cloud Storage`_. - Required external libraries: `google-cloud-storage`_. -For more information about authentication, please refer to `Google Cloud documentation `_. +For more information about authentication, please refer to `Google Cloud documentation `_. You can set a *Project ID* and *Access Control List (ACL)* through the following settings: @@ -261,7 +261,7 @@ storage backend is: ``True``. This storage backend uses :ref:`delayed file delivery `. -.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python +.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python .. _topics-feed-storage-stdout: diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 769925dd5..8d6b2458b 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -104,6 +104,15 @@ If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. It is ``False`` by default. +.. note:: Because requests are serialized with :mod:`pickle`, the objects you + store on a request, such as the values of its + :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` + dictionaries, are deep-copied when the request is written to and later read + back from the job directory. As a result, the callback receives a *copy* of + those objects rather than the original ones, and changes made to the copy are + not reflected in the original object. Keep this in mind if you rely on + sharing mutable state through ``cb_kwargs`` or ``meta``. + .. _job-dir-contents: Job directory contents diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index e61f33aed..0913a3310 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -68,19 +68,21 @@ Response, Item, Spider and Selector objects. You can enter the telnet console and inspect how many objects (of the classes mentioned above) are currently alive using the ``prefs()`` function which is an -alias to the :func:`~scrapy.utils.trackref.print_live_refs` function:: +alias to the :func:`~scrapy.utils.trackref.print_live_refs` function: + +.. code-block:: bash telnet localhost 6023 - .. code-block:: pycon +.. code-block:: pycon - >>> prefs() - Live References + >>> prefs() + Live References - ExampleSpider 1 oldest: 15s ago - HtmlResponse 10 oldest: 1s ago - Selector 2 oldest: 0s ago - FormRequest 878 oldest: 7s ago + ExampleSpider 1 oldest: 15s ago + HtmlResponse 10 oldest: 1s ago + Selector 2 oldest: 0s ago + Request 878 oldest: 7s ago As you can see, that report also shows the "age" of the oldest object in each class. If you're running multiple spiders per process chances are you can diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 7e3517937..613e175da 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -159,5 +159,3 @@ Link :synopsis: Link from link extractors .. autoclass:: Link - -.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index a43edb28c..3ccc90bf9 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -102,14 +102,13 @@ One approach to overcome this is to define items using the .. code-block:: python from dataclasses import dataclass, field - from typing import Optional @dataclass class InventoryItem: - name: Optional[str] = field(default=None) - price: Optional[float] = field(default=None) - stock: Optional[int] = field(default=None) + name: str | None = field(default=None) + price: float | None = field(default=None) + stock: int | None = field(default=None) .. _topics-loaders-processors: @@ -228,7 +227,8 @@ metadata. Here is an example: .. code-block:: python - import scrapy + from dataclasses import dataclass, field + from itemloaders.processors import Join, MapCompose, TakeFirst from w3lib.html import remove_tags @@ -238,14 +238,21 @@ metadata. Here is an example: return value - class Product(scrapy.Item): - name = scrapy.Field( - input_processor=MapCompose(remove_tags), - output_processor=Join(), + @dataclass + class Product: + name: str | None = field( + default=None, + metadata={ + "input_processor": MapCompose(remove_tags), + "output_processor": Join(), + }, ) - price = scrapy.Field( - input_processor=MapCompose(remove_tags, filter_price), - output_processor=TakeFirst(), + price: str | None = field( + default=None, + metadata={ + "input_processor": MapCompose(remove_tags, filter_price), + "output_processor": TakeFirst(), + }, ) @@ -452,4 +459,3 @@ organization of your Loaders collection - that's up to you and your project's needs. .. _itemloaders: https://itemloaders.readthedocs.io/en/latest/ -.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index a398d6c83..bbb5d0458 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -194,6 +194,48 @@ If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy component that prints the log. It is unset by default, hence logs contain the Scrapy component responsible for that log output. +Rotating log files +------------------ + +Scrapy's :setting:`LOG_FILE` setting writes logs to a single file. It does not +rotate log files automatically, but you can use Python's standard +:mod:`logging.handlers` module when running Scrapy from a script. + +For example, to rotate the log file every day: + +.. skip: next + +.. code-block:: python + + import logging + from logging.handlers import TimedRotatingFileHandler + + from scrapy.crawler import CrawlerProcess + from scrapy.utils.project import get_project_settings + + from myproject.spiders.myspider import MySpider + + settings = get_project_settings() + process = CrawlerProcess(settings, install_root_handler=False) + + handler = TimedRotatingFileHandler( + "scrapy.log", + when="midnight", + backupCount=7, + encoding=settings.get("LOG_ENCODING"), + ) + handler.setFormatter( + logging.Formatter(settings.get("LOG_FORMAT"), settings.get("LOG_DATEFORMAT")) + ) + + root_logger = logging.getLogger() + root_logger.setLevel(settings.get("LOG_LEVEL")) + root_logger.addHandler(handler) + + process.crawl(MySpider) + process.start() + + Command-line options -------------------- diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 8636e29c9..c67d2d627 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -81,9 +81,6 @@ thumbnailing and normalizing images to JPEG/RGB format. Enabling your Media Pipeline ============================ -.. setting:: IMAGES_STORE -.. setting:: FILES_STORE - To enable your media pipeline you must first add it to your project :setting:`ITEM_PIPELINES` setting. @@ -102,6 +99,8 @@ For Files Pipeline, use: .. note:: You can also use both the Files and Images Pipeline at the same time. +.. setting:: IMAGES_STORE +.. setting:: FILES_STORE Then, configure the target storage setting to a valid value that will be used for storing the downloaded images. Otherwise the pipeline will remain disabled, @@ -290,7 +289,7 @@ Google Cloud Storage :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) -.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python +.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings: @@ -301,7 +300,7 @@ For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_I For information about authentication, see this `documentation`_. -.. _documentation: https://cloud.google.com/docs/authentication +.. _documentation: https://docs.cloud.google.com/docs/authentication You can modify the Access Control List (ACL) policy used for the stored files, which is defined by the :setting:`FILES_STORE_GCS_ACL` and @@ -316,7 +315,7 @@ policy: For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide. -.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl +.. _Predefined ACLs: https://docs.cloud.google.com/storage/docs/access-control/lists#predefined-acl Usage example ============= @@ -337,17 +336,18 @@ respectively), the pipeline will put the results under the respective field When using :ref:`item types ` for which fields are defined beforehand, you must define both the URLs field and the results field. For example, when using the images pipeline, items must define both the ``image_urls`` and the -``images`` field. For instance, using the :class:`~scrapy.Item` class: +``images`` field. For instance, using a dataclass: .. code-block:: python - import scrapy + from dataclasses import dataclass, field - class MyItem(scrapy.Item): + @dataclass + class MyItem: # ... other item fields ... - image_urls = scrapy.Field() - images = scrapy.Field() + image_urls: list[str] = field(default_factory=list) + images: list[dict] = field(default_factory=list) If you want to use another field name for the URLs key or for the results key, it is also possible to override it. @@ -774,4 +774,28 @@ To enable your custom media pipeline component you must add its class import pat ITEM_PIPELINES = {"myproject.pipelines.MyImagesPipeline": 300} +Content-based image filtering pipeline +-------------------------------------- + +This example overrides ``get_images()`` to filter images using a classifier, +such as a TensorFlow_ model. Override ``is_valid_image()`` with your +classification logic: + +.. code-block:: python + + from scrapy.pipelines.images import ImagesPipeline, ImageException + + + class ImageClassifierPipeline(ImagesPipeline): + def is_valid_image(self, image): + raise NotImplementedError + + def get_images(self, response, request, info, *, item=None): + for path, image, buf in super().get_images(response, request, info, item=item): + if not self.is_valid_image(image): + raise ImageException("Image does not match criteria") + yield path, image, buf + + .. _MD5 hash: https://en.wikipedia.org/wiki/MD5 +.. _TensorFlow: https://tensorflow.org diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 79d5bcce9..8a04f7ada 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -166,6 +166,86 @@ with :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`): .. seealso:: :doc:`twisted:core/howto/reactor-basics` +And here are examples of using these classes with +:setting:`TWISTED_REACTOR_ENABLED` set to ``False``. + +Simple usage of :class:`~scrapy.crawler.AsyncCrawlerProcess`: + +.. code-block:: python + + import scrapy + from scrapy.crawler import AsyncCrawlerProcess + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + } + ) + + process.crawl(MySpider) + process.start() # the script will block here until the crawling is finished + +With ``TWISTED_REACTOR_ENABLED=False`` you can use several instances of +:class:`~scrapy.crawler.AsyncCrawlerProcess` in the same process: + +.. code-block:: python + + import scrapy + from scrapy.crawler import AsyncCrawlerProcess + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + process1 = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + } + ) + process1.crawl(MySpider) + process1.start() + + process2 = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR_ENABLED": False, + } + ) + process2.crawl(MySpider) + process2.start() + +Using :func:`asyncio.run` with :class:`~scrapy.crawler.AsyncCrawlerRunner`: + +.. code-block:: python + + import asyncio + + import scrapy + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.log import configure_logging + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + async def main(): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) + await runner.crawl(MySpider) # completes when the spider finishes + + + asyncio.run(main()) + + .. _run-multiple-spiders: Running multiple spiders in the same process @@ -307,6 +387,26 @@ crawl:: curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2 curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3 +.. _large-project-startup: + +Reducing startup time in large projects +======================================= + +When running a spider with ``scrapy crawl``, Scrapy loads all modules listed in +:setting:`SPIDER_MODULES` to find the target spider. In large projects with +many spiders, this can noticeably increase startup time and memory usage. + +To avoid loading every spider module, override :setting:`SPIDER_MODULES` on the +command line to point only to the module that contains the spider you want to +run: + +.. code-block:: shell + + scrapy crawl myspider -s SPIDER_MODULES=myproject.spiders.myspider + +Because :setting:`SPIDER_MODULES` is a list setting, you can include multiple +modules by separating them with commas. + .. _bans: Avoiding getting banned @@ -329,6 +429,10 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * use a pool of rotating IPs. For example, the free `Tor project`_ or paid services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. +* for HTTPS websites, if blocking appears related to TLS behavior, consider + adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond + differently depending on the TLS method used by the client. * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy plugin `__ and additional features, like `AI web scraping `__ @@ -336,8 +440,16 @@ Here are some tips to keep in mind when dealing with these kinds of sites: If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. +.. _static-analysis: + +Static analysis +=============== + +Consider using :doc:`scrapy-lint `, a linter for Scrapy +projects that detects common mistakes and anti-patterns. + .. _Tor project: https://www.torproject.org/ -.. _commercial support: https://scrapy.org/support/ +.. _commercial support: https://www.scrapy.org/companies .. _ProxyMesh: https://proxymesh.com/ .. _Common Crawl: https://commoncrawl.org/ .. _testspiders: https://github.com/scrapinghub/testspiders diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index cb2410207..700238fe9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -117,6 +117,9 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the body to bytes (if given as a string). + + To disable URL percent-encoding for a request, use the + :reqmeta:`verbatim_url` request meta key. :type encoding: str :param priority: sets :attr:`priority`, defaults to ``0``. @@ -136,9 +139,13 @@ Request objects .. attribute:: Request.url - A string containing the URL of this request. Keep in mind that this - attribute contains the escaped URL, so it can differ from the URL passed in - the ``__init__()`` method. + A string containing the URL of this request. + + Keep in mind that this attribute contains the escaped URL, so it can + differ from the URL passed in the ``__init__()`` method. + + If :reqmeta:`verbatim_url` is set to ``True``, the URL is kept as + passed to ``__init__()``. This attribute is read-only. To change the URL of a Request use :meth:`replace`. @@ -181,6 +188,13 @@ Request objects ``failure.request.cb_kwargs`` in the request's errback. For more information, see :ref:`errback-cb_kwargs`. + .. note:: When :setting:`JOBDIR` is set, requests are serialized to disk + with :mod:`pickle` (see :ref:`request-serialization`). As a result, + the callback receives a deep copy of any object stored in + ``cb_kwargs``, so mutating such an object in the callback does not + affect the original. Avoid relying on shared mutable state passed + through ``cb_kwargs`` in that case. + .. attribute:: Request.meta :value: {} @@ -246,6 +260,78 @@ Request objects .. automethod:: to_dict +.. _form: + +Creating requests that submit HTML forms +---------------------------------------- + +Use :doc:`form2request ` to build request data from an HTML +``
`` element and convert it to a :class:`~scrapy.Request`. + +Install it with pip: + +.. code-block:: bash + + pip install form2request + +Select the desired form with CSS or XPath, then build and convert request +data: + +.. code-block:: python + + from form2request import form2request + + + def parse(self, response): + form = response.css("form#search") + request_data = form2request(form, data={"q": "scrapy"}) + yield request_data.to_scrapy(callback=self.parse_results) + +Use ``data`` to override field values. To drop a field from the resulting +request, set its value to ``None``. + +By default, form2request simulates clicking the first submit button. To submit +without clicking any button, pass ``click=False``. To click a specific submit +button, pass its element: + +.. code-block:: python + + def parse(self, response): + form = response.css("form#checkout") + submit = form.css('button[name="pay"]') + request_data = form2request(form, click=submit) + +.. _topics-request-response-ref-request-userlogin: + +Using form2request to simulate a user login +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is usual for web sites to provide pre-populated form fields through ```` elements, such as session related data or authentication +tokens (for login pages). Build the request from the form and only override the +credentials: + +.. code-block:: python + + import scrapy + from form2request import form2request + + + class LoginSpider(scrapy.Spider): + name = "example.com" + start_urls = ["http://www.example.com/users/login.php"] + + def parse(self, response): + form = response.css("form") + request_data = form2request( + form, + data={"username": "john", "password": "secret"}, + ) + yield request_data.to_scrapy(callback=self.after_login) + + def after_login(self, response): ... + + Other functions related to requests ----------------------------------- @@ -469,6 +555,11 @@ in your :meth:`fingerprint` method implementation: .. autofunction:: scrapy.utils.request.fingerprint +By default, request fingerprinting canonicalizes the request URL. If +:reqmeta:`verbatim_url` is set to ``True``, fingerprinting does not +canonicalize the URL, and the ``keep_fragments`` parameter is ignored (it is +effectively true). + For example, to take the value of a request header named ``X-ID`` into account: @@ -630,21 +721,59 @@ Those are: * :reqmeta:`download_timeout` * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * ``ftp_user`` (See :setting:`FTP_USER` for more info) +* :reqmeta:`give_up_log_level` * :reqmeta:`handle_httpstatus_all` * :reqmeta:`handle_httpstatus_list` +* :reqmeta:`http_auth_domain` +* :reqmeta:`http_pass` +* :reqmeta:`http_user` * :reqmeta:`is_start_request` * :reqmeta:`max_retry_times` * :reqmeta:`proxy` * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` +* :reqmeta:`verbatim_url` .. reqmeta:: bindaddress bindaddress ----------- -The IP of the outgoing IP address to use for the performing the request. +The default local outgoing address for download-handler connections. + +This meta value can be either: + +- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local + port is chosen automatically, or + +- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a + specific local interface and a specific local port. + +For example: + +.. code-block:: python + + Request( + "https://example.org", + meta={"bindaddress": "127.0.0.2"}, + ) + +.. code-block:: python + + Request( + "https://example.org", + meta={"bindaddress": ("127.0.0.2", 50000)}, + ) + +If not set, built-in HTTP download handlers use the value of +:setting:`DOWNLOAD_BIND_ADDRESS` as the default bind address. +Set the :reqmeta:`bindaddress` request meta key to override it for a +specific request. + +This meta key is not supported by +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`, but the +:setting:`DOWNLOAD_BIND_ADDRESS` is supported by it. .. reqmeta:: download_timeout @@ -672,15 +801,59 @@ download_fail_on_dataloss Whether or not to fail on broken responses. See: :setting:`DOWNLOAD_FAIL_ON_DATALOSS`. +.. reqmeta:: give_up_log_level + +give_up_log_level +----------------- + +:ref:`Logging level ` used for the message logged when a request +exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details. + +.. reqmeta:: http_auth_domain + +http_auth_domain +---------------- + +Overrides :setting:`HTTPAUTH_DOMAIN` for this request. + +.. reqmeta:: http_pass + +http_pass +--------- + +Overrides :setting:`HTTPAUTH_PASS` for this request. + +.. reqmeta:: http_user + +http_user +--------- + +Overrides :setting:`HTTPAUTH_USER` for this request. + .. reqmeta:: max_retry_times max_retry_times --------------- -The meta key is used set retry times per request. When initialized, the +The meta key is used set retry times per request. When set, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. +.. reqmeta:: verbatim_url + +verbatim_url +------------ + +Set this key to ``True`` to keep the request URL as passed to +:class:`~scrapy.Request`, without URL percent-encoding. + +When this key is enabled, :func:`~scrapy.utils.request.fingerprint` does not +canonicalize the request URL, so requests whose URLs differ only in +characters that would otherwise be canonicalized get different fingerprints. + +In this mode, the ``keep_fragments`` parameter is ignored, and it is +effectively true. + .. _topics-stop-response-download: @@ -738,159 +911,6 @@ Request subclasses Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass it to implement your own custom functionality. -FormRequest objects -------------------- - -The FormRequest class extends the base :class:`~scrapy.Request` with functionality for -dealing with HTML forms. It uses `lxml.html forms`_ to pre-populate form -fields with form data from :class:`Response` objects. - -.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms - -.. currentmodule:: None - -.. class:: scrapy.FormRequest(url, [formdata, ...]) - :canonical: scrapy.http.request.form.FormRequest - - The :class:`~scrapy.FormRequest` class adds a new keyword parameter to the ``__init__()`` method. The - remaining arguments are the same as for the :class:`~scrapy.Request` class and are - not documented here. - - :param formdata: is a dictionary (or iterable of (key, value) tuples) - containing HTML Form data which will be url-encoded and assigned to the - body of the request. - :type formdata: dict or collections.abc.Iterable - - The :class:`~scrapy.FormRequest` objects support the following class method in - addition to the standard :class:`~scrapy.Request` methods: - - .. classmethod:: from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) - - Returns a new :class:`~scrapy.FormRequest` object with its form field values - pre-populated with those found in the HTML ```` element contained - in the given response. For an example see - :ref:`topics-request-response-ref-request-userlogin`. - - The policy is to automatically simulate a click, by default, on any form - control that looks clickable, like a ````. Even - though this is quite convenient, and often the desired behaviour, - sometimes it can cause problems which could be hard to debug. For - example, when working with forms that are filled and/or submitted using - javascript, the default :meth:`from_response` behaviour may not be the - most appropriate. To disable this behaviour you can set the - ``dont_click`` argument to ``True``. Also, if you want to change the - control clicked (instead of disabling it) you can also use the - ``clickdata`` argument. - - .. caution:: Using this method with select elements which have leading - or trailing whitespace in the option values will not work due to a - `bug in lxml`_, which should be fixed in lxml 3.8 and above. - - :param response: the response containing a HTML form which will be used - to pre-populate the form fields - :type response: :class:`~scrapy.http.Response` object - - :param formname: if given, the form with name attribute set to this value will be used. - :type formname: str - - :param formid: if given, the form with id attribute set to this value will be used. - :type formid: str - - :param formxpath: if given, the first form that matches the xpath will be used. - :type formxpath: str - - :param formcss: if given, the first form that matches the css selector will be used. - :type formcss: str - - :param formnumber: the number of form to use, when the response contains - multiple forms. The first one (and also the default) is ``0``. - :type formnumber: int - - :param formdata: fields to override in the form data. If a field was - already present in the response ```` element, its value is - overridden by the one passed in this parameter. If a value passed in - this parameter is ``None``, the field will not be included in the - request, even if it was present in the response ```` element. - :type formdata: dict - - :param clickdata: attributes to lookup the control clicked. If it's not - given, the form data will be submitted simulating a click on the - first clickable element. In addition to html attributes, the control - can be identified by its zero-based index relative to other - submittable inputs inside the form, via the ``nr`` attribute. - :type clickdata: dict - - :param dont_click: If True, the form data will be submitted without - clicking in any element. - :type dont_click: bool - - The other parameters of this class method are passed directly to the - :class:`~scrapy.FormRequest` ``__init__()`` method. - -.. currentmodule:: scrapy.http - -Request usage examples ----------------------- - -Using FormRequest to send data via HTTP POST -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want to simulate a HTML Form POST in your spider and send a couple of -key-value fields, you can return a :class:`~scrapy.FormRequest` object (from your -spider) like this: - -.. skip: next -.. code-block:: python - - return [ - FormRequest( - url="http://www.example.com/post/action", - formdata={"name": "John Doe", "age": "27"}, - callback=self.after_post, - ) - ] - -.. _topics-request-response-ref-request-userlogin: - -Using FormRequest.from_response() to simulate a user login -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is usual for web sites to provide pre-populated form fields through ```` elements, such as session related data or authentication -tokens (for login pages). When scraping, you'll want these fields to be -automatically pre-populated and only override a couple of them, such as the -user name and password. You can use the :meth:`.FormRequest.from_response` -method for this job. Here's an example spider which uses it: - -.. code-block:: python - - import scrapy - - - def authentication_failed(response): - # TODO: Check the contents of the response and return True if it failed - # or False if it succeeded. - pass - - - class LoginSpider(scrapy.Spider): - name = "example.com" - start_urls = ["http://www.example.com/users/login.php"] - - def parse(self, response): - return scrapy.FormRequest.from_response( - response, - formdata={"username": "john", "password": "secret"}, - callback=self.after_login, - ) - - def after_login(self, response): - if authentication_failed(response): - self.logger.error("Login failed") - return - - # continue scraping with authenticated session... - JsonRequest ----------- @@ -965,7 +985,7 @@ Response objects :type request: scrapy.Request :param certificate: an object representing the server's SSL certificate. - :type certificate: twisted.internet.ssl.Certificate + :type certificate: typing.Any :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` @@ -990,7 +1010,7 @@ Response objects A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains the response headers. Values can be accessed using - :meth:`~scrapy.http.headers.Headers.get` to return the first header value with + :meth:`~scrapy.http.headers.Headers.get` to return the last header value with the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return all header values with the specified name. For example, this call will give you all cookies in the headers:: @@ -1057,8 +1077,8 @@ Response objects .. attribute:: Response.certificate - A :class:`twisted.internet.ssl.Certificate` object representing - the server's SSL certificate. + An object representing the server's SSL certificate. Its type and + contents depend on the download handler that produced the response. Only populated for ``https`` responses, ``None`` otherwise. diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 40a85201a..35ea2a200 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -543,7 +543,7 @@ you may want to take a look first at this `XPath tutorial`_. .. note:: Some of the tips are based on `this post from Zyte's blog`_. -.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html +.. _XPath tutorial: http://www.zvon.org/comp/r/tut-XPath_1.html .. _this post from Zyte's blog: https://www.zyte.com/blog/xpath-tips-from-the-web-scraping-trenches/ @@ -728,7 +728,7 @@ But using the ``.`` to mean the node, works: >>> sel.xpath("//a[contains(., 'Next Page')]").getall() ['Click here to go to the Next Page'] -.. _`XPath string function`: https://www.w3.org/TR/xpath-10/#section-String-Functions +.. _XPath string function: https://www.w3.org/TR/xpath-10/#section-String-Functions .. _topics-selectors-xpath-variables: @@ -983,9 +983,9 @@ Here we first iterate over ``itemscope`` elements, and for each one, we look for all ``itemprops`` elements and exclude those that are themselves inside another ``itemscope``. -.. _EXSLT: http://exslt.org/ -.. _regular expressions: http://exslt.org/regexp/index.html -.. _set manipulation: http://exslt.org/set/index.html +.. _EXSLT: https://exslt.github.io/ +.. _regular expressions: https://exslt.github.io/regexp/index.html +.. _set manipulation: https://exslt.github.io/set/index.html Other XPath extensions ---------------------- @@ -1190,4 +1190,4 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object: .. skip: end -.. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799 +.. _Google Base XML feed: https://support.google.com/merchants/answer/14987622 diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5b4a9b5f9..455985a56 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -303,11 +303,12 @@ Pre-crawler settings These settings cannot be :ref:`set from a spider `. -These settings are :setting:`SPIDER_LOADER_CLASS` and settings used by the -corresponding :ref:`component `, e.g. -:setting:`SPIDER_MODULES` and :setting:`SPIDER_LOADER_WARN_ONLY` for the -default component. +These settings are: +- :setting:`TWISTED_REACTOR_ENABLED` +- :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding + spider loader class, e.g. :setting:`SPIDER_MODULES` and + :setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class. .. _reactor-settings: @@ -331,7 +332,7 @@ These settings are: - :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) -- :setting:`DNS_RESOLVER` and settings used by the corresponding +- :setting:`TWISTED_DNS_RESOLVER` and settings used by the corresponding component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE` and :setting:`DNS_TIMEOUT` for the default one. @@ -356,6 +357,9 @@ ignoring the value of :setting:`TWISTED_REACTOR` and using the value of e.g. in :ref:`per-spider settings `, an exception will be raised. +All of these settings, except for :setting:`ASYNCIO_EVENT_LOOP`, are only used +when the Twisted reactor is used, i.e. when :setting:`TWISTED_REACTOR_ENABLED` +is ``True``. .. _topics-settings-ref: @@ -651,6 +655,13 @@ Default: ``True`` Whether to enable DNS in-memory cache. +.. note:: + This setting is only used by + :class:`~scrapy.resolver.CachingThreadedResolver` and + :class:`~scrapy.resolver.CachingHostnameResolver`. It has no effect when + :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect + either when :setting:`DNS_RESOLVER` is set to a different resolver. + .. setting:: DNSCACHE_SIZE DNSCACHE_SIZE @@ -658,21 +669,25 @@ DNSCACHE_SIZE Default: ``10000`` -DNS in-memory cache size. +DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`. -.. setting:: DNS_RESOLVER +.. setting:: TWISTED_DNS_RESOLVER -DNS_RESOLVER ------------- +TWISTED_DNS_RESOLVER +-------------------- Default: ``'scrapy.resolver.CachingThreadedResolver'`` -The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver`` -supports specifying a timeout for DNS requests via the :setting:`DNS_TIMEOUT` setting, -but works only with IPv4 addresses. Scrapy provides an alternative resolver, +The class to be used by Twisted to resolve DNS names. The default +``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for +DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4 +addresses. Scrapy provides an alternative resolver, ``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not take the :setting:`DNS_TIMEOUT` setting into account. +.. note:: + This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. + .. setting:: DNS_TIMEOUT DNS_TIMEOUT @@ -682,6 +697,12 @@ Default: ``60`` Timeout for processing of DNS queries in seconds. Float is supported. +.. note:: + This setting is only used by + :class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when + :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect + either when :setting:`DNS_RESOLVER` is set to a different resolver. + .. setting:: DOWNLOADER DOWNLOADER @@ -691,41 +712,6 @@ Default: ``'scrapy.core.downloader.Downloader'`` The downloader to use for crawling. -.. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY - -DOWNLOADER_CLIENTCONTEXTFACTORY -------------------------------- - -Default: ``'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'`` - -Represents the classpath to the ContextFactory to use. - -Here, "ContextFactory" is a Twisted term for SSL/TLS contexts, defining -the TLS/SSL protocol version to use, whether to do certificate verification, -or even enable client-side authentication (and various other things). - -.. note:: - - Scrapy default context factory **does NOT perform remote server - certificate verification**. This is usually fine for web scraping. - - If you do need remote server certificate verification enabled, - Scrapy also has another context factory class that you can set, - ``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``, - which uses the platform's certificates to validate remote endpoints. - -If you do use a custom ContextFactory, make sure its ``__init__`` method -accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping -:setting:`DOWNLOADER_CLIENT_TLS_METHOD`), a ``tls_verbose_logging`` -parameter (``bool``) and a ``tls_ciphers`` parameter (see -:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`). - -.. note:: - - This setting is specific to the built-in Twisted-based download handlers: - :class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and - :class:`scrapy.core.downloader.handlers.http2.H2DownloadHandler`. - .. setting:: DOWNLOADER_CLIENT_TLS_CIPHERS DOWNLOADER_CLIENT_TLS_CIPHERS @@ -748,41 +734,52 @@ specific cipher that is not included in ``DEFAULT`` if a website requires it. Handling of this setting needs to be implemented inside the :ref:`download handler `, so it's not guaranteed to be supported - by all 3rd-party handlers. Moreover, for the built-in Twisted-based - download handlers - (:class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and - :class:`scrapy.core.downloader.handlers.http2.H2DownloadHandler`) it needs - to be implemented in the :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` class. + by all 3rd-party handlers. -.. setting:: DOWNLOADER_CLIENT_TLS_METHOD +.. setting:: DOWNLOAD_TLS_MAX_VERSION -DOWNLOADER_CLIENT_TLS_METHOD ----------------------------- +DOWNLOAD_TLS_MAX_VERSION +------------------------ -Default: ``'TLS'`` +Default: ``None`` -Use this setting to customize the TLS/SSL method used by the HTTPS download -handler. +Use this setting to change the maximum version of the TLS protocol allowed to +be used by Scrapy. -This setting must be one of these string values: +This setting must be either ``None``, in which case it doesn't affect the +version selection, or one of these string values: -- ``'TLS'``: maps to OpenSSL's ``TLS_method()`` (a.k.a ``SSLv23_method()``), - which allows protocol negotiation, starting from the highest supported - by the platform; **default, recommended** -- ``'TLSv1.0'``: this value forces HTTPS connections to use TLS version 1.0 ; - set this if you want the behavior of Scrapy<1.1 -- ``'TLSv1.1'``: forces TLS version 1.1 -- ``'TLSv1.2'``: forces TLS version 1.2 +- ``'TLSv1.0'`` +- ``'TLSv1.1'`` +- ``'TLSv1.2'`` +- ``'TLSv1.3'`` + +The range of allowed TLS versions advertised by Scrapy when making TLS +connections will depend on the TLS implementation defaults and the values of +:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`. +It's possible to re-enable versions that are supported by the TLS +implementation but disabled by default by adjusting these settings, but it's +impossible to enable unsupported ones, such as any versions below 1.2 in many +modern environments. .. note:: Handling of this setting needs to be implemented inside the :ref:`download handler `, so it's not guaranteed to be supported - by all 3rd-party handlers. Moreover, for the built-in Twisted-based - download handlers - (:class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and - :class:`scrapy.core.downloader.handlers.http2.H2DownloadHandler`) it needs - to be implemented in the :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` class. + by all 3rd-party handlers. Additionally, the set of supported TLS versions + depends on the TLS implementation being used by the handler. + +.. setting:: DOWNLOAD_TLS_MIN_VERSION + +DOWNLOAD_TLS_MIN_VERSION +------------------------ + +Default: ``None`` + +Use this setting to change the minimum version of the TLS protocol allowed to +be used by Scrapy. + +See :setting:`DOWNLOAD_TLS_MAX_VERSION` for the details and limitations. .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING @@ -800,18 +797,14 @@ the TLS-related libraries. Handling of this setting needs to be implemented inside the :ref:`download handler `, so it's not guaranteed to be supported - by all 3rd-party handlers. Moreover, for the built-in Twisted-based - download handlers - (:class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and - :class:`scrapy.core.downloader.handlers.http2.H2DownloadHandler`) it needs - to be implemented in the :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` class. + by all 3rd-party handlers. .. setting:: DOWNLOADER_MIDDLEWARES DOWNLOADER_MIDDLEWARES ---------------------- -Default:: ``{}`` +Default: ``{}`` A dict containing the downloader middlewares enabled in your project, and their orders. For more info see :ref:`topics-downloader-middleware-setting`. @@ -833,7 +826,6 @@ Default: "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400, "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500, "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550, - "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560, "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, @@ -897,6 +889,46 @@ It is also possible to change this setting per domain, although it requires non-trivial code. See the implementation of the :ref:`AutoThrottle ` extension for an example. +.. setting:: DOWNLOAD_BIND_ADDRESS + +DOWNLOAD_BIND_ADDRESS +--------------------- + +Default: ``None`` + +The default local outgoing address for download-handler connections. + +This setting can be either: + +- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local + port is chosen automatically, or + +- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a + specific local interface and a specific local port. + +For example: + +.. code-block:: python + + # Bind to this local address + DOWNLOAD_BIND_ADDRESS = "127.0.0.2" + +.. code-block:: python + + # Bind to this local address and local port + DOWNLOAD_BIND_ADDRESS = ("127.0.0.2", 5000) + +If set, built-in HTTP download handlers use this value by default. +Set the :reqmeta:`bindaddress` request meta key to override it for a specific +request. + +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. Specifying the port is unsupported by + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + .. setting:: DOWNLOAD_HANDLERS DOWNLOAD_HANDLERS @@ -927,6 +959,20 @@ Default: "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler", } +(when :setting:`TWISTED_REACTOR_ENABLED` is ``True``) + +.. code-block:: python + + { + "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler", + "file": "scrapy.core.downloader.handlers.file.FileDownloadHandler", + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler", + "ftp": None, + } + +(when :setting:`TWISTED_REACTOR_ENABLED` is ``False``) A dict containing the :ref:`download handlers ` enabled by default in Scrapy. You should never modify this setting in your @@ -1082,6 +1128,24 @@ Optionally, this can be set per-request basis by using the requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])`` failure is always raised for every request that was using that connection. +.. setting:: DOWNLOAD_VERIFY_CERTIFICATES + +DOWNLOAD_VERIFY_CERTIFICATES +---------------------------- + +Default: ``False`` + +Whether the HTTPS download handlers should verify the server TLS certificate +when making a request and abort the request if the verification fails. + +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. The exact behavior of a handler (e.g. whether + certificate problems are logged when this setting is set to ``False``) + depends on its implementation. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS @@ -1191,7 +1255,7 @@ command will prefer it over the default setting. EXTENSIONS ---------- -Default:: ``{}`` +Default: ``{}`` :ref:`Component priority dictionary ` of enabled extensions. See :ref:`topics-extensions`. @@ -1239,7 +1303,7 @@ FEED_STORAGE_GCS_ACL -------------------- The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. -For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. +For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. .. setting:: FORCE_CRAWLER_PROCESS @@ -1529,13 +1593,12 @@ MEMUSAGE_ENABLED Default: ``True`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` Whether to enable the memory usage extension. This extension keeps track of a peak memory used by the process (it writes it to stats). It can also optionally shutdown the Scrapy process when it exceeds a memory limit -(see :setting:`MEMUSAGE_LIMIT_MB`), and notify by email when that happened -(see :setting:`MEMUSAGE_NOTIFY_MAIL`). +(see :setting:`MEMUSAGE_LIMIT_MB`). See :ref:`topics-extensions-ref-memusage`. @@ -1546,10 +1609,11 @@ MEMUSAGE_LIMIT_MB Default: ``0`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` The maximum amount of memory to allow (in megabytes) before shutting down -Scrapy (if MEMUSAGE_ENABLED is True). If zero, no check will be performed. +Scrapy (if :setting:`MEMUSAGE_ENABLED` is ``True``). If zero, no check will be +performed. See :ref:`topics-extensions-ref-memusage`. @@ -1560,7 +1624,7 @@ MEMUSAGE_CHECK_INTERVAL_SECONDS Default: ``60.0`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` The :ref:`Memory usage extension ` checks the current memory usage, versus the limits set by @@ -1571,23 +1635,6 @@ This sets the length of these intervals, in seconds. See :ref:`topics-extensions-ref-memusage`. -.. setting:: MEMUSAGE_NOTIFY_MAIL - -MEMUSAGE_NOTIFY_MAIL --------------------- - -Default: ``False`` - -Scope: ``scrapy.extensions.memusage`` - -A list of emails to notify if the memory limit has been reached. - -Example:: - - MEMUSAGE_NOTIFY_MAIL = ['user@example.com'] - -See :ref:`topics-extensions-ref-memusage`. - .. setting:: MEMUSAGE_WARNING_MB MEMUSAGE_WARNING_MB @@ -1595,10 +1642,13 @@ MEMUSAGE_WARNING_MB Default: ``0`` -Scope: ``scrapy.extensions.memusage`` +Scope: ``scrapy.extensions.memusage.MemoryUsage`` -The maximum amount of memory to allow (in megabytes) before sending a warning -email notifying about it. If zero, no warning will be produced. +The maximum amount of memory to allow (in megabytes) before sending a +:signal:`memusage_warning_reached` signal (if :setting:`MEMUSAGE_ENABLED` is +``True``). If zero, no signal will be sent. + +See :ref:`topics-extensions-ref-memusage`. .. setting:: NEWSPIDER_MODULE @@ -1827,7 +1877,7 @@ Scrapy does not process new requests. SPIDER_CONTRACTS ---------------- -Default:: ``{}`` +Default: ``{}`` A dict containing the spider contracts enabled in your project, used for testing spiders. For more info see :ref:`topics-contracts`. @@ -1888,7 +1938,7 @@ warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. SPIDER_MIDDLEWARES ------------------ -Default:: ``{}`` +Default: ``{}`` A dict containing the spider middlewares enabled in your project, and their orders. For more info see :ref:`topics-spider-middleware-setting`. @@ -1950,22 +2000,12 @@ finishes. For more info see: :ref:`topics-stats`. -.. setting:: STATSMAILER_RCPTS - -STATSMAILER_RCPTS ------------------ - -Default: ``[]`` (empty list) - -Send Scrapy stats after spiders finish scraping. See -:class:`~scrapy.extensions.statsmailer.StatsMailer` for more info. - .. setting:: TELNETCONSOLE_ENABLED TELNETCONSOLE_ENABLED --------------------- -Default: ``True`` +Default: ``True`` (``False`` when :setting:`TWISTED_REACTOR_ENABLED` is ``False``) A boolean which specifies if the :ref:`telnet console ` will be enabled (provided its extension is also enabled). @@ -1984,6 +2024,35 @@ command. The project name must not conflict with the name of custom files or directories in the ``project`` subdirectory. +.. setting:: TWISTED_REACTOR_ENABLED + +TWISTED_REACTOR_ENABLED +----------------------- + +Default: ``True`` + +Whether to install and use the Twisted reactor. + +If this is set to ``True``, Scrapy will use the Twisted reactor and will +install one according to the :setting:`TWISTED_REACTOR` setting value when +appropriate (e.g. when running via :ref:`the command-line tool +`). This is the traditional mode of using Scrapy. + +If this is set to ``False``, Scrapy will use the asyncio event loop directly +and will not attempt to install or use a reactor. Features that require a +reactor won't be available, but Twisted APIs that don't require a reactor, +including :class:`~twisted.internet.defer.Deferred` and +:class:`~twisted.python.failure.Failure`, will still be available. On the other +hand, limitations related to Twisted reactors (such as not being able to start +a reactor in the same process where a reactor was previously started and +stopped) will not apply. This mode is currently experimental and may not be +suitable for production use. It may also not be supported by 3rd-party code. +See :ref:`asyncio-without-reactor` for more information about this mode. + +.. note:: This setting can't be set :ref:`per-spider `. + +.. versionadded:: 2.15.0 + .. setting:: TWISTED_REACTOR TWISTED_REACTOR @@ -2109,7 +2178,7 @@ Use ``0`` to allow URLs of any length. The default value is copied from the `Microsoft Internet Explorer maximum URL length`_, even though this setting exists for different reasons. -.. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 +.. _Microsoft Internet Explorer maximum URL length: https://web.archive.org/web/20250206050143/https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 .. setting:: USER_AGENT @@ -2148,6 +2217,4 @@ case to see how to enable and use them. .. settingslist:: .. _Amazon web services: https://aws.amazon.com/ -.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search -.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search .. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 8ae8ff512..5b826ed18 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -40,7 +40,7 @@ variable; or by defining it in your :ref:`scrapy.cfg `:: shell = bpython .. _IPython: https://ipython.org/ -.. _IPython installation guide: https://ipython.org/install.html +.. _IPython installation guide: https://ipython.org/install/ .. _bpython: https://bpython-interpreter.org/ Launch the shell @@ -111,7 +111,7 @@ Available Shortcuts Note, however, that this will create a temporary file in your computer, which won't be removed automatically. -.. _ tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/base +.. _ tag: https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/base Available Scrapy objects ------------------------ @@ -145,7 +145,7 @@ Example of shell session .. skip: start Here's an example of a typical shell session where we start by scraping the -https://scrapy.org page, and then proceed to scrape the https://old.reddit.com/ +https://www.scrapy.org/ page, and then proceed to scrape the https://old.reddit.com/ page. Finally, we modify the (Reddit) request method to POST and re-fetch it getting an error. We end the session by typing Ctrl-D (in Unix systems) or Ctrl-Z in Windows. diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index efe3bb8fc..d13733623 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -356,6 +356,18 @@ feed_exporter_closed This signal supports :ref:`asynchronous handlers `. +memusage_warning_reached +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. signal:: memusage_warning_reached + +.. function:: memusage_warning_reached() + + Sent by the :class:`~scrapy.extensions.memusage.MemoryUsage` extension when the + memory usage reaches the warning threshold (:setting:`MEMUSAGE_WARNING_MB`). + + This signal does not support :ref:`asynchronous handlers `. + Request signals --------------- diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 799d2f99c..99bbdf292 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -117,36 +117,28 @@ one or more of these methods: :type response: :class:`~scrapy.http.Response` object .. method:: process_spider_output(response, result) + :async: - This method is called with the results returned from the Spider, after - it has processed the response. + This method is an :term:`asynchronous generator` called with the + results from the spider after the spider has processed the response. - :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.Request` objects and :ref:`item objects - `. - - Consider defining this method as an :term:`asynchronous generator`, - which will be a requirement in a future version of Scrapy. However, if - you plan on sharing your spider middleware with other people, consider - either :ref:`enforcing Scrapy 2.7 ` - as a minimum requirement of your spider middleware, or :ref:`making - your spider middleware universal ` so that - it works with Scrapy versions earlier than Scrapy 2.7. + .. seealso:: :ref:`universal-spider-middleware`. :param response: the response which generated this output from the spider :type response: :class:`~scrapy.http.Response` object - :param result: the result returned by the spider - :type result: an iterable of :class:`~scrapy.Request` objects and - :ref:`item objects ` + :param result: the results from the spider + :type result: an :term:`asynchronous iterable` of + :class:`~scrapy.Request` objects and :ref:`item objects + ` .. method:: process_spider_output_async(response, result) :async: - If defined, this method must be an :term:`asynchronous generator`, - which will be called instead of :meth:`process_spider_output` if - ``result`` is an :term:`asynchronous iterable`. + Alternative name for :meth:`process_spider_output` used when + implementing a :ref:`universal spider middleware + `. .. method:: process_spider_exception(response, exception) @@ -174,13 +166,40 @@ one or more of these methods: :type exception: :exc:`Exception` object +.. _universal-spider-middleware: + +Universal spider middlewares +---------------------------- + +In Scrapy 2.6.3 and lower, ``process_spider_output()`` must be a *synchronous* +generator. + +To support those versions and higher Scrapy versions in the same middleware, +rename your asynchronous :meth:`~SpiderMiddleware.process_spider_output` +method to :meth:`~SpiderMiddleware.process_spider_output_async`, and define a +synchronous ``process_spider_output()`` method to be used by 2.6.3 and lower +versions. + +For example: + +.. code-block:: python + + class UniversalSpiderMiddleware: + async def process_spider_output_async(self, response, result): + async for r in result: + # ... do something with r + yield r + + def process_spider_output(self, response, result): + for r in result: + # ... do something with r + yield r + Base class for custom spider middlewares ---------------------------------------- Scrapy provides a base class for custom spider middlewares. It's not required -to use it but it can help with simplifying middleware implementations and -reducing the amount of boilerplate code in :ref:`universal middlewares -`. +to use it but it can help with simplifying middleware implementations. .. module:: scrapy.spidermiddlewares.base @@ -403,6 +422,25 @@ String value Class name (as a string) .. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin .. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url +.. setting:: REFERRER_POLICIES + +REFERRER_POLICIES +^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.14.2 + +Default: ``{}`` + +A dictionary mapping policy names to import paths of +:class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` subclasses, or +``None`` to disable support for a given policy name. + +This allows overriding the policies triggered by the ``Referrer-Policy`` +response header. + +Use ``""`` to override the policy for responses with `no referrer policy +`__. + StartSpiderMiddleware --------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 98374b6b4..506daf930 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -198,7 +198,7 @@ scrapy.Spider The ``parse`` method is in charge of processing the response and returning scraped data and/or more URLs to follow. Other Requests callbacks have - the same requirements as the :class:`Spider` class. + the same requirements as the :class:`~scrapy.Spider` class. This method, as well as any other Request callback, must return a :class:`~scrapy.Request` object, an :ref:`item object `, an @@ -354,11 +354,6 @@ Otherwise, you would cause iteration over a ``start_urls`` string (a very common python pitfall) resulting in each character being seen as a separate url. -A valid use case is to set the http auth credentials -used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`:: - - scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword - Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. @@ -457,13 +452,14 @@ with a ``TestItem`` declared in a ``myproject.items`` module: .. code-block:: python - import scrapy + from dataclasses import dataclass - class TestItem(scrapy.Item): - id = scrapy.Field() - name = scrapy.Field() - description = scrapy.Field() + @dataclass + class TestItem: + id: str | None = None + name: str | None = None + description: str | None = None .. currentmodule:: scrapy.spiders @@ -556,7 +552,6 @@ Let's now take a look at an example CrawlSpider with rules: .. code-block:: python - import scrapy from scrapy.spiders import CrawlSpider, Rule from scrapy.linkextractors import LinkExtractor @@ -576,7 +571,7 @@ Let's now take a look at an example CrawlSpider with rules: def parse_item(self, response): self.logger.info("Hi, this is an item page! %s", response.url) - item = scrapy.Item() + item = {} item["id"] = response.xpath('//td[@id="item_id"]/text()').re(r"ID: (\d+)") item["name"] = response.xpath('//td[@id="item_name"]/text()').get() item["description"] = response.xpath( @@ -714,9 +709,9 @@ These spiders are pretty easy to use, let's have a look at one example: ) item = TestItem() - item["id"] = node.xpath("@id").get() - item["name"] = node.xpath("name").get() - item["description"] = node.xpath("description").get() + item.id = node.xpath("@id").get() + item.name = node.xpath("name").get() + item.description = node.xpath("description").get() return item Basically what we did up there was to create a spider that downloads a feed from @@ -778,9 +773,9 @@ Let's see an example similar to the previous one, but using a self.logger.info("Hi, this is a row!: %r", row) item = TestItem() - item["id"] = row["id"] - item["name"] = row["name"] - item["description"] = row["description"] + item.id = row["id"] + item.name = row["name"] + item.description = row["description"] return item diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index ae9cb634c..e274edc9b 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -26,6 +26,9 @@ disable it if you want. For more information about the extension itself see Please avoid using telnet console over insecure connections, or disable it completely using :setting:`TELNETCONSOLE_ENABLED` option. +.. note:: + This feature is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. + .. highlight:: none How to access the telnet console @@ -43,12 +46,12 @@ the console you need to type:: Password: >>> -By default Username is ``scrapy`` and Password is autogenerated. The -autogenerated Password can be seen on Scrapy logs like the example below:: +By default, the username is ``scrapy`` and the password is autogenerated. The +autogenerated password can be seen on Scrapy logs like the example below:: 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 -Default Username and Password can be overridden by the settings +The default username and password can be overridden by the settings :setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`. .. warning:: @@ -91,8 +94,6 @@ convenience: +----------------+-------------------------------------------------------------------+ | ``p`` | a shortcut to the :func:`pprint.pprint` function | +----------------+-------------------------------------------------------------------+ -| ``hpy`` | for memory debugging (see :ref:`topics-leaks`) | -+----------------+-------------------------------------------------------------------+ Telnet console usage examples ============================= diff --git a/docs/versioning.rst b/docs/versioning.rst index aab6af865..0c6498244 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -23,7 +23,7 @@ Development releases do not follow 3-numbers version and are generally released as ``dev`` suffixed versions, e.g. ``1.3dev``. .. note:: - With Scrapy 0.* series, Scrapy used `odd-numbered versions for development releases`_. + With Scrapy 0.* series, Scrapy used odd-numbered versions for development releases. This is not the case anymore from Scrapy 1.0 onwards. Starting with Scrapy 1.0, all releases should be considered production-ready. @@ -63,6 +63,3 @@ feature. All deprecated features removed in a Scrapy release are explicitly mentioned in the :ref:`release notes `. - - -.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 734614aa5..569438574 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -18,7 +18,7 @@ class Root(Resource): self.tail.clear() self.start = self.lastmark = self.lasttime = time() - def getChild(self, request, name): + def getChild(self, path, request): return self def render(self, request): diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 269b27336..8e5001c1d 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -35,10 +35,6 @@ class QPSSpider(Spider): self.download_delay = float(self.download_delay) async def start(self): - for item_or_request in self.start_requests(): - yield item_or_request - - def start_requests(self): url = self.benchurl if self.latency is not None: url += f"?latency={self.latency}" diff --git a/pyproject.toml b/pyproject.toml index 5138fda88..6b0c561f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,8 +7,7 @@ name = "Scrapy" dynamic = ["version"] description = "A high-level Web Crawling and Web Scraping framework" dependencies = [ - # Twisted pinned until Scrapy is updated for its internal TLS API changes - "Twisted>=21.7.0,<=25.5.0", + "Twisted>=21.7.0", "cryptography>=37.0.0", "cssselect>=0.9.1", "defusedxml>=0.7.1", @@ -20,7 +19,7 @@ dependencies = [ "protego>=0.1.15", "pyOpenSSL>=22.0.0", "queuelib>=1.4.2", - "service_identity>=18.1.0", + "service_identity>=23.1.0", "tldextract", "w3lib>=1.17.0", "zope.interface>=5.1.0", @@ -40,6 +39,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet :: WWW/HTTP", @@ -86,13 +86,10 @@ pattern = "^(?P.+)$" [tool.mypy] strict = true -allow_any_generics = true # 67 errors -allow_untyped_calls = true # 58 errors extra_checks = false # weird addErrback() errors untyped_calls_exclude = [ "twisted", ] -warn_return_any = false # 37 errors [[tool.mypy.overrides]] module = "tests.*" @@ -123,24 +120,9 @@ implicit_reexport = true module = "scrapy.settings.default_settings" ignore_errors = true -# deprecated modules -[[tool.mypy.overrides]] -module = [ - "scrapy.core.downloader.webclient", - "scrapy.spiders.init", - "scrapy.utils.testsite", - "tests.test_webclient", -] -allow_any_generics = true -allow_untyped_calls = true -allow_untyped_defs = true -check_untyped_defs = false -warn_return_any = false - # usually no type hints [[tool.mypy.overrides]] module = [ -# "IPython.*", "bpython", "brotli", "brotlicffi", @@ -155,7 +137,7 @@ module = [ ignore_missing_imports = true [tool.bumpversion] -current_version = "2.14.1" +current_version = "2.16.0" commit = true tag = true tag_name = "{new_version}" @@ -175,6 +157,8 @@ parse = """(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)""" serialize = ["{major}.{minor}"] [tool.coverage.run] +# sysmon, default on 3.14, is too slow: https://github.com/coveragepy/coveragepy/issues/2172 +core = "ctrace" branch = true include = ["scrapy/*"] omit = ["tests/*"] @@ -201,6 +185,7 @@ jobs = 1 # >1 hides results extension-pkg-allow-list=[ "lxml", ] +load-plugins = ["pylint_per_file_ignores"] [tool.pylint."MESSAGES CONTROL"] enable = [ @@ -242,8 +227,10 @@ disable = [ "too-many-positional-arguments", "too-many-public-methods", "too-many-return-statements", + "undefined-variable", "unused-argument", "unused-variable", + "use-implicit-booleaness-not-comparison", "useless-import-alias", # used as a hint to mypy "useless-return", # https://github.com/pylint-dev/pylint/issues/6530 "wrong-import-position", @@ -260,15 +247,13 @@ disable = [ "unused-import", # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") - "abstract-method", "arguments-differ", - "arguments-renamed", - "dangerous-default-value", "keyword-arg-before-vararg", - "pointless-statement", - "raise-missing-from", - "unnecessary-dunder-call", - "used-before-assignment", +] +# requires `pylint_per_file_ignores` plugin +per-file-ignores = [ + # Extended list of ones that we may want to address, only for tests + "./tests/*:abstract-method,arguments-renamed,dangerous-default-value,pointless-statement,raise-missing-from,unnecessary-dunder-call,used-before-assignment", ] [tool.pytest.ini_options] @@ -285,6 +270,7 @@ markers = [ "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", "requires_mitmproxy: marks tests that need mitmproxy", + "requires_internet: marks tests that need real Internet access", ] filterwarnings = [ "ignore::DeprecationWarning:twisted.web.static", @@ -416,25 +402,6 @@ ignore = [ "SIM115", # Yoda condition detected "SIM300", - - # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") - - # Assigning to `os.environ` doesn't clear the environment. - "B003", - # Do not use mutable data structures for argument defaults. - "B006", - # Found useless expression. - "B018", - # No explicit stacklevel argument found. - "B028", - # Within an `except` clause, raise exceptions with `raise ... from` - "B904", - # `for` loop variable overwritten by assignment target - "PLW2901", - # Mutable class attributes should be annotated with `typing.ClassVar` - "RUF012", - # Use capitalized environment variable - "SIM112", ] [tool.ruff.lint.flake8-tidy-imports] @@ -454,8 +421,28 @@ split-on-trailing-comma = false "scrapy/linkextractors/__init__.py" = ["E402"] "scrapy/spiders/__init__.py" = ["E402"] -# Skip bandit and allow blocking file I/O in tests -"tests/**" = ["ASYNC240", "S"] +"tests/**" = [ + # Skip bandit and allow blocking file I/O in tests + "ASYNC240", + "S", + # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") + # Assigning to `os.environ` doesn't clear the environment. + "B003", + # Do not use mutable data structures for argument defaults. + "B006", + # Found useless expression. + "B018", + # No explicit stacklevel argument found. + "B028", + # Within an `except` clause, raise exceptions with `raise ... from` + "B904", + # `for` loop variable overwritten by assignment target + "PLW2901", + # Mutable class attributes should be annotated with `typing.ClassVar` + "RUF012", + # Use capitalized environment variable + "SIM112", +] # Issues pending a review: "docs/conf.py" = ["E402"] @@ -464,3 +451,6 @@ split-on-trailing-comma = false [tool.ruff.lint.pydocstyle] convention = "pep257" + +[tool.sphinx-scrapy] +python-version = "3.14" # Keep in sync with .github/workflows/checks.yml. diff --git a/scrapy/VERSION b/scrapy/VERSION index b70ae75a8..752490696 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.14.1 +2.16.0 diff --git a/scrapy/__main__.py b/scrapy/__main__.py index 697b9b1e9..17b1aa538 100644 --- a/scrapy/__main__.py +++ b/scrapy/__main__.py @@ -1,3 +1,4 @@ +# pragma: no file cover from scrapy.cmdline import execute if __name__ == "__main__": diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 487980abd..6c306afdb 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -12,7 +12,7 @@ import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import AsyncCrawlerProcess, CrawlerProcess from scrapy.exceptions import UsageError -from scrapy.utils.misc import walk_modules +from scrapy.utils.misc import walk_modules_iter from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect from scrapy.utils.reactor import _asyncio_reactor_path @@ -40,13 +40,13 @@ class ScrapyArgumentParser(argparse.ArgumentParser): def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]: # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes - for module in walk_modules(module_name): + for module in walk_modules_iter(module_name): for obj in vars(module).values(): if ( inspect.isclass(obj) and issubclass(obj, ScrapyCommand) and obj.__module__ == module.__name__ - and obj not in (ScrapyCommand, BaseRunSpiderCommand) + and obj not in {ScrapyCommand, BaseRunSpiderCommand} ): yield obj @@ -108,17 +108,24 @@ def _print_header(settings: BaseSettings, inproject: bool) -> None: def _print_commands(settings: BaseSettings, inproject: bool) -> None: _print_header(settings, inproject) - print("Usage:") - print(" scrapy [options] [args]\n") - print("Available commands:") + print( + "Usage:\n", + " scrapy [options] [args]\n", + "Available commands:\n", + ) cmds = _get_commands_dict(settings, inproject) - for cmdname, cmdclass in sorted(cmds.items()): - print(f" {cmdname:<13} {cmdclass.short_desc()}") + print( + "\n".join( + f" {cmdname:<13} {cmdclass.short_desc()}" + for cmdname, cmdclass in sorted(cmds.items()) + ) + ) if not inproject: - print() - print(" [ more ] More commands available when run from project directory") - print() - print('Use "scrapy -h" to see more info about a command') + print( + "\n", + " [ more ] More commands available when run from project directory", + ) + print("\n", 'Use "scrapy -h" to see more info about a command') def _print_unknown_command_msg( @@ -197,9 +204,10 @@ def execute(argv: list[str] | None = None, settings: Settings | None = None) -> _run_print_help(parser, cmd.process_options, args, opts) if cmd.requires_crawler_process: - if settings[ - "TWISTED_REACTOR" - ] == _asyncio_reactor_path and not settings.getbool("FORCE_CRAWLER_PROCESS"): + if ( + settings["TWISTED_REACTOR"] == _asyncio_reactor_path + and not settings.getbool("FORCE_CRAWLER_PROCESS") + ) or not settings.getbool("TWISTED_REACTOR_ENABLED"): cmd.crawler_process = AsyncCrawlerProcess(settings) else: cmd.crawler_process = CrawlerProcess(settings) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d5945f6f5..598e8060e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -7,14 +7,17 @@ from __future__ import annotations import argparse import builtins import os +import warnings from abc import ABC, abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from twisted.python import failure -from scrapy.exceptions import UsageError +from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.utils.deprecate import method_is_overridden +from scrapy.utils.python import global_object_name if TYPE_CHECKING: from collections.abc import Iterable @@ -29,14 +32,27 @@ class ScrapyCommand(ABC): crawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline # default settings to be used for this command instead of global defaults - default_settings: dict[str, Any] = {} + default_settings: ClassVar[dict[str, Any]] = {} exitcode: int = 0 def __init__(self) -> None: self.settings: Settings | None = None # set in scrapy.cmdline + if method_is_overridden(self.__class__, ScrapyCommand, "help"): + warnings.warn( + "The ScrapyCommand.help() method is deprecated and overriding " + f"it, as the {global_object_name(self.__class__)} class does, " + "has no effect; override long_desc() instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) - def set_crawler(self, crawler: Crawler) -> None: + def set_crawler(self, crawler: Crawler) -> None: # pragma: no cover + warnings.warn( + "ScrapyCommand.set_crawler() is deprecated", + ScrapyDeprecationWarning, + stacklevel=2, + ) if hasattr(self, "_crawler"): raise RuntimeError("crawler already set") self._crawler: Crawler = crawler @@ -62,10 +78,11 @@ class ScrapyCommand(ABC): return self.short_desc() def help(self) -> str: - """An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines since no post-formatting will - be applied to its contents. - """ + warnings.warn( + "ScrapyCommand.help() is deprecated, use long_desc() instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) return self.long_desc() def add_options(self, parser: argparse.ArgumentParser) -> None: @@ -109,7 +126,9 @@ class ScrapyCommand(ABC): try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: - raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False) + raise UsageError( + "Invalid -s value, use -s NAME=VALUE", print_help=False + ) from None if opts.logfile: self.settings.set("LOG_ENABLED", True, priority="cmdline") @@ -175,7 +194,9 @@ class BaseRunSpiderCommand(ScrapyCommand): try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) + raise UsageError( + "Invalid -a value, use -a NAME=VALUE", print_help=False + ) from None if opts.output or opts.overwrite_output: assert self.settings is not None feeds = feed_process_params_from_cli( @@ -219,7 +240,7 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): headings = [ i for i in range(len(part_strings)) if part_strings[i].endswith(":\n") ] - for index in headings[::-1]: + for index in reversed(headings): char = "-" if "Global Options" in part_strings[index] else "=" part_strings[index] = part_strings[index][:-2].title() underline = "".join(["\n", (char * len(part_strings[index])), "\n"]) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index cc39d344a..0a5b431a2 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -3,7 +3,7 @@ from __future__ import annotations import subprocess import sys import time -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from urllib.parse import urlencode import scrapy @@ -18,7 +18,7 @@ if TYPE_CHECKING: class Command(ScrapyCommand): - default_settings = { + default_settings: ClassVar[dict[str, Any]] = { "LOG_LEVEL": "INFO", "LOGSTATS_INTERVAL": 1, "CLOSESPIDER_TIMEOUT": 10, diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 17e66e20c..2113c19d2 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -2,7 +2,7 @@ import argparse import time from collections import defaultdict from collections.abc import AsyncIterator -from typing import Any +from typing import Any, ClassVar from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner @@ -44,7 +44,7 @@ class TextTestResult(_TextTestResult): class Command(ScrapyCommand): requires_project = True - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return "[options] " @@ -73,7 +73,9 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: # load contracts assert self.settings is not None - contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) + contracts = build_component_list( + self.settings.get_component_priority_dict_with_base("SPIDER_CONTRACTS") + ) conman = ContractsManager(load_object(c) for c in contracts) runner = TextTestRunner(verbosity=2 if opts.verbose else 1) result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity) @@ -102,16 +104,18 @@ class Command(ScrapyCommand): # start checks if opts.list: - for spider, methods in sorted(contract_reqs.items()): - if not methods and not opts.verbose: - continue - print(spider) - for method in sorted(methods): - print(f" * {method}") + print( + "\n".join( + f"{spider}\n" + + "\n".join(f" * {method}" for method in sorted(methods)) + for spider, methods in sorted(contract_reqs.items()) + if methods or opts.verbose + ) + ) else: - start_time = time.time() + start_time = time.monotonic() self.crawler_process.start() - stop = time.time() + stop = time.monotonic() result.printErrors() result.printSummary(start_time, stop) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index f2d52673a..cd7c57f28 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,6 +1,7 @@ import argparse import os import sys +from typing import Any, ClassVar from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError @@ -10,7 +11,7 @@ from scrapy.spiderloader import get_spider_loader class Command(ScrapyCommand): requires_project = True requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return "" diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 9d2742afd..cc8624fa1 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -5,7 +5,7 @@ import shutil import string from importlib import import_module from pathlib import Path -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, ClassVar, cast from urllib.parse import urlparse import scrapy @@ -47,7 +47,7 @@ def verify_url_scheme(url: str) -> str: class Command(ScrapyCommand): requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return "[options] " @@ -147,7 +147,7 @@ class Command(ScrapyCommand): name: str, url: str, template_name: str, - template_file: str | os.PathLike, + template_file: str | os.PathLike[str], ) -> None: """Generate the spider module, based on the given template""" assert self.settings is not None @@ -173,15 +173,21 @@ class Command(ScrapyCommand): template_file = Path(self.templates_dir, f"{template}.tmpl") if template_file.exists(): return template_file - print(f"Unable to find template: {template}\n") - print('Use "scrapy genspider --list" to see all available templates.') + print( + f"Unable to find template: {template}\n", + 'Use "scrapy genspider --list" to see all available templates.', + ) return None def _list_templates(self) -> None: - print("Available templates:") - for file in sorted(Path(self.templates_dir).iterdir()): - if file.suffix == ".tmpl": - print(f" {file.stem}") + print( + "Available templates:\n", + "\n".join( + f" {file.stem}" + for file in sorted(Path(self.templates_dir).iterdir()) + if file.suffix == ".tmpl" + ), + ) def _spider_exists(self, name: str) -> bool: assert self.settings is not None @@ -200,8 +206,10 @@ class Command(ScrapyCommand): pass else: # if spider with same name exists - print(f"Spider {name!r} already exists in module:") - print(f" {spidercls.__module__}") + print( + f"Spider {name!r} already exists in module:\n", + f" {spidercls.__module__}", + ) return True # a file with the same name exists in the target directory diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index b4dc97f3d..a037f5c9b 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar from scrapy.commands import ScrapyCommand from scrapy.spiderloader import get_spider_loader @@ -12,7 +12,7 @@ if TYPE_CHECKING: class Command(ScrapyCommand): requires_project = True requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def short_desc(self) -> str: return "List available spiders" @@ -20,5 +20,4 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: assert self.settings is not None spider_loader = get_spider_loader(self.settings) - for s in sorted(spider_loader.list()): - print(s) + print("\n".join(sorted(spider_loader.list()))) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index e1e027c95..2ac65bf3f 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -4,7 +4,7 @@ import functools import inspect import json import logging -from typing import TYPE_CHECKING, Any, TypeVar, overload +from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, overload from itemadapter import ItemAdapter from twisted.internet.defer import Deferred, maybeDeferred @@ -39,8 +39,8 @@ class Command(BaseRunSpiderCommand): requires_project = True spider: Spider | None = None - items: dict[int, list[Any]] = {} - requests: dict[int, list[Request]] = {} + items: ClassVar[dict[int, list[Any]]] = {} + requests: ClassVar[dict[int, list[Request]]] = {} spidercls: type[Spider] | None first_response = None @@ -144,17 +144,16 @@ class Command(BaseRunSpiderCommand): def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... def iterate_spider_output(self, result: Any) -> Iterable[Any] | Deferred[Any]: + d: Deferred[Any] if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) ) - d.addCallback(self.iterate_spider_output) - return d + return d.addCallback(self.iterate_spider_output) + d = deferred_from_coro(result) if inspect.iscoroutine(result): - d = deferred_from_coro(result) - d.addCallback(self.iterate_spider_output) - return d - return arg_to_iter(deferred_from_coro(result)) + return d.addCallback(self.iterate_spider_output) + return arg_to_iter(d) def add_items(self, lvl: int, new_items: list[Any]) -> None: old_items = self.items.get(lvl, []) @@ -387,7 +386,7 @@ class Command(BaseRunSpiderCommand): "Invalid -m/--meta value, pass a valid json string to -m or --meta. " 'Example: --meta=\'{"foo" : "bar"}\'', print_help=False, - ) + ) from None def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None: if opts.cbkwargs: @@ -398,7 +397,7 @@ class Command(BaseRunSpiderCommand): "Invalid --cbkwargs value, pass a valid json string to --cbkwargs. " 'Example: --cbkwargs=\'{"foo" : "bar"}\'', print_help=False, - ) + ) from None def run(self, args: list[str], opts: argparse.Namespace) -> None: # parse arguments diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index eeb1303e2..0b9036457 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -3,7 +3,7 @@ from __future__ import annotations import sys from importlib import import_module from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -18,7 +18,7 @@ if TYPE_CHECKING: def _import_file(filepath: str | PathLike[str]) -> ModuleType: abspath = Path(filepath).resolve() - if abspath.suffix not in (".py", ".pyw"): + if abspath.suffix not in {".py", ".pyw"}: raise ValueError(f"Not a Python source file: {abspath}") dirname = str(abspath.parent) sys.path = [dirname, *sys.path] @@ -30,7 +30,9 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: class Command(BaseRunSpiderCommand): - default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader} + default_settings: ClassVar[dict[str, Any]] = { + "SPIDER_LOADER_CLASS": DummySpiderLoader + } def syntax(self) -> str: return "[options] " @@ -50,7 +52,7 @@ class Command(BaseRunSpiderCommand): try: module = _import_file(filename) except (ImportError, ValueError) as e: - raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") + raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") from e spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError(f"No spider found in file: {filename}\n") diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 704cc500d..26a97ccbb 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,5 +1,6 @@ import argparse import json +from typing import Any, ClassVar from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings @@ -7,7 +8,7 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return "[options]" diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 080f62382..19138ffd0 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -6,10 +6,12 @@ See documentation in docs/topics/shell.rst from __future__ import annotations +import asyncio from threading import Thread -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from scrapy.commands import ScrapyCommand +from scrapy.crawler import AsyncCrawlerProcess, Crawler from scrapy.http import Request from scrapy.shell import Shell from scrapy.utils.defer import _schedule_coro @@ -23,7 +25,7 @@ if TYPE_CHECKING: class Command(ScrapyCommand): - default_settings = { + default_settings: ClassVar[dict[str, Any]] = { "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, "LOGSTATS_INTERVAL": 0, @@ -83,16 +85,47 @@ class Command(ScrapyCommand): # crawling engine, so the set up in the crawl method won't work crawler = self.crawler_process._create_crawler(spidercls) crawler._apply_settings() - # The Shell class needs a persistent engine in the crawler - crawler.engine = crawler._create_engine() - _schedule_coro(crawler.engine.start_async(_start_request_processing=False)) - - self._start_crawler_thread() - - shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) + loop: asyncio.AbstractEventLoop | None = None + if crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): + self._init_with_reactor(crawler) + else: + self._init_without_reactor(crawler) + loop = self._get_reactorless_loop() + shell = Shell(crawler, update_vars=self.update_vars, code=opts.code, loop=loop) shell.start(url=url, redirect=not opts.no_redirect) + def _init_with_reactor(self, crawler: Crawler) -> None: + # Create the engine and run start_async() in the main thread + crawler.engine = crawler._create_engine() + _schedule_coro(crawler.engine.start_async(_start_request_processing=False)) + self._start_crawler_thread() + + def _init_without_reactor(self, crawler: Crawler) -> None: + # Create the engine and run start_async() in the event loop thread + loop = self._get_reactorless_loop() + self._start_crawler_thread() + + async def _init_engine() -> None: + # We may need to wait until some parts of start_async() have + # finished, which may need a special event in the engine and may + # wait until https://github.com/scrapy/scrapy/issues/6916 + crawler.engine = crawler._create_engine() + loop.create_task( + crawler.engine.start_async(_start_request_processing=False) + ) + + future = asyncio.run_coroutine_threadsafe(_init_engine(), loop) + future.result() + + def _get_reactorless_loop(self) -> asyncio.AbstractEventLoop: + assert self.crawler_process + assert isinstance(self.crawler_process, AsyncCrawlerProcess) + loop = self.crawler_process._reactorless_loop + assert loop + return loop + def _start_crawler_thread(self) -> None: + """Run self.crawler_process.start() in a separate thread.""" assert self.crawler_process t = Thread( target=self.crawler_process.start, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 8f4427580..5661a0a48 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -6,7 +6,7 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar import scrapy from scrapy.commands import ScrapyCommand @@ -34,7 +34,7 @@ def _make_writable(path: Path) -> None: class Command(ScrapyCommand): requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return " [project_dir]" @@ -90,7 +90,7 @@ class Command(ScrapyCommand): _make_writable(dst) def run(self, args: list[str], opts: argparse.Namespace) -> None: - if len(args) not in (1, 2): + if len(args) not in {1, 2}: raise UsageError project_name = args[0] @@ -123,12 +123,12 @@ class Command(ScrapyCommand): ) print( f"New Scrapy project '{project_name}', using template directory " - f"'{self.templates_dir}', created in:" + f"'{self.templates_dir}', created in:\n", + f" {project_dir.resolve()}\n\n", + "You can start your first spider with:\n", + f" cd {project_dir}\n", + " scrapy genspider example example.com", ) - print(f" {project_dir.resolve()}\n") - print("You can start your first spider with:") - print(f" cd {project_dir}") - print(" scrapy genspider example example.com") @property def templates_dir(self) -> str: diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 30b0e9fd7..1d1985d4e 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,4 +1,5 @@ import argparse +from typing import Any, ClassVar import scrapy from scrapy.commands import ScrapyCommand @@ -7,7 +8,7 @@ from scrapy.utils.versions import get_versions class Command(ScrapyCommand): requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return "[-v]" diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index cbdb36d2f..ebbdf1b98 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -6,7 +6,7 @@ from collections.abc import AsyncGenerator, Iterable from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, ClassVar, cast from unittest import TestCase, TestResult from scrapy.http import Request, Response @@ -27,7 +27,7 @@ class Contract: request_cls: type[Request] | None = None name: str - def __init__(self, method: Callable, *args: Any): + def __init__(self, method: Callable[..., Any], *args: Any): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") self.args: tuple[Any, ...] = args @@ -51,6 +51,8 @@ class Contract: results.addSuccess(self.testcase_pre) cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + if isinstance(cb_result, CoroutineType): + cb_result.close() raise TypeError("Contracts don't support async callbacks") return list(cast("Iterable[Any]", iterate_spider_output(cb_result))) @@ -67,6 +69,8 @@ class Contract: def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + if isinstance(cb_result, CoroutineType): + cb_result.close() raise TypeError("Contracts don't support async callbacks") output = list(cast("Iterable[Any]", iterate_spider_output(cb_result))) try: @@ -90,7 +94,7 @@ class Contract: class ContractsManager: - contracts: dict[str, type[Contract]] = {} + contracts: ClassVar[dict[str, type[Contract]]] = {} def __init__(self, contracts: Iterable[type[Contract]]): for contract in contracts: @@ -105,11 +109,11 @@ class ContractsManager: return methods - def extract_contracts(self, method: Callable) -> list[Contract]: + def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]: contracts: list[Contract] = [] assert method.__doc__ is not None - for line in method.__doc__.split("\n"): - line = line.strip() + for line_ in method.__doc__.split("\n"): + line = line_.strip() if line.startswith("@"): m = re.match(r"@(\w+)\s*(.*)", line) @@ -125,7 +129,7 @@ class ContractsManager: def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]: requests: list[Request | None] = [] for method in self.tested_methods_from_spidercls(type(spider)): - bound_method = spider.__getattribute__(method) + bound_method = getattr(spider, method) try: requests.append(self.from_method(bound_method, results)) except Exception: @@ -134,7 +138,9 @@ class ContractsManager: return requests - def from_method(self, method: Callable, results: TestResult) -> Request | None: + def from_method( + self, method: Callable[..., Any], results: TestResult + ) -> Request | None: contracts = self.extract_contracts(method) if contracts: request_cls = Request @@ -170,7 +176,7 @@ class ContractsManager: return None def _clean_req( - self, request: Request, method: Callable, results: TestResult + self, request: Request, method: Callable[..., Any], results: TestResult ) -> None: """stop the request from returning objects and records any errors""" @@ -195,7 +201,7 @@ class ContractsManager: request.errback = eb_wrapper -def _create_testcase(method: Callable, desc: str) -> TestCase: +def _create_testcase(method: Callable[..., Any], desc: str) -> TestCase: spider = method.__self__.name # type: ignore[attr-defined] class ContractTestCase(TestCase): diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 90f054a87..9b42ca36f 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from itemadapter import ItemAdapter, is_item @@ -68,7 +68,7 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers: dict[str | None, Callable[[Any], bool]] = { + object_type_verifiers: ClassVar[dict[str | None, Callable[[Any], bool]]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, @@ -78,7 +78,7 @@ class ReturnsContract(Contract): def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - if len(self.args) not in [1, 2, 3]: + if len(self.args) not in {1, 2, 3}: raise ValueError( f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}" ) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 59871fdc3..7c0ee0eec 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -2,8 +2,9 @@ from __future__ import annotations import random from collections import deque +from dataclasses import dataclass, field from datetime import datetime -from time import time +from time import monotonic from typing import TYPE_CHECKING, Any from twisted.internet.defer import Deferred, inlineCallbacks @@ -40,24 +41,21 @@ if TYPE_CHECKING: from scrapy.signalmanager import SignalManager +@dataclass(slots=True, eq=False) class Slot: """Downloader slot""" - def __init__( - self, - concurrency: int, - delay: float, - randomize_delay: bool, - ): - self.concurrency: int = concurrency - self.delay: float = delay - self.randomize_delay: bool = randomize_delay + concurrency: int + delay: float + randomize_delay: bool - self.active: set[Request] = set() - self.queue: deque[tuple[Request, Deferred[Response]]] = deque() - self.transferring: set[Request] = set() - self.lastseen: float = 0 - self.latercall: CallLaterResult | None = None + active: set[Request] = field(default_factory=set, init=False, repr=False) + queue: deque[tuple[Request, Deferred[Response]]] = field( + default_factory=deque, init=False, repr=False + ) + transferring: set[Request] = field(default_factory=set, init=False, repr=False) + lastseen: float = field(default=0, init=False, repr=False) + latercall: CallLaterResult | None = field(default=None, init=False, repr=False) def free_transfer_slots(self) -> int: return self.concurrency - len(self.transferring) @@ -72,14 +70,6 @@ class Slot: self.latercall.cancel() self.latercall = None - def __repr__(self) -> str: - cls_name = self.__class__.__name__ - return ( - f"{cls_name}(concurrency={self.concurrency!r}, " - f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r})" - ) - def __str__(self) -> str: return ( f" Generator[Deferred[Any], Any, Response | Request]: self.active.add(request) try: - return ( - yield deferred_from_coro( + result: Response | Request = yield ( + deferred_from_coro( self.middleware.download_async(self._enqueue_request, request) ) ) + return result finally: self.active.remove(request) @@ -173,7 +164,8 @@ class Downloader: return key, self.slots[key] def get_slot_key(self, request: Request) -> str: - if (meta_slot := request.meta.get(self.DOWNLOAD_SLOT)) is not None: + meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) + if meta_slot is not None: return meta_slot key = urlparse_cached(request).hostname or "" @@ -206,7 +198,7 @@ class Downloader: return # Delay queue processing if a download_delay is configured - now = time() + now = monotonic() delay = slot.download_delay() if delay: penalty = delay - now + slot.lastseen @@ -275,7 +267,7 @@ class Downloader: slot.close() def _slot_gc(self, age: float = 60) -> None: - mintime = time() - age + mintime = monotonic() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: self.slots.pop(key).close() diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 252f509bf..40f09ffb2 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,15 +1,14 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from OpenSSL import SSL -from twisted.internet._sslverify import _setAcceptableProtocols from twisted.internet.ssl import ( AcceptableCiphers, CertificateOptions, + TLSVersion, optionsForClientTLS, - platformTrust, ) from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS @@ -17,13 +16,17 @@ from zope.interface.declarations import implementer from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import ( + _TWISTED_VERSION_MAP, DEFAULT_CIPHERS, - ScrapyClientTLSOptions, - openssl_methods, + _openssl_methods, + _ScrapyClientTLSOptions, + _ScrapyClientTLSOptions26, ) from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.deprecate import method_is_overridden +from scrapy.utils._deps_compat import TWISTED_TLS_NEW_IMPL +from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.ssl import _get_cert_options_version_kwargs, _get_tls_version_limits if TYPE_CHECKING: from twisted.internet._sslverify import ClientTLSOptions @@ -36,46 +39,47 @@ if TYPE_CHECKING: @implementer(IPolicyForHTTPS) -class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): - """ - Non-peer-certificate verifying HTTPS context factory +class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): + """Non-peer-certificate verifying HTTPS context factory. - Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) - which allows TLS protocol negotiation + Uses :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`, + :setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION` + to configure the :class:`~twisted.internet.ssl.CertificateOptions` + instance. - 'A TLS/SSL connection established with [this method] may - understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.' + The purpose of this custom class is to provide a ``creatorForNetloc()`` + method that returns a ``_ScrapyClientTLSOptions`` instance configured based + on TLS settings provided to the factory. """ def __init__( self, - method: int = SSL.SSLv23_METHOD, # noqa: S503 + method: int | None = SSL.SSLv23_METHOD, # noqa: S503 tls_verbose_logging: bool = False, tls_ciphers: str | None = None, *args: Any, + verify_certificates: bool = False, + tls_min_version: TLSVersion | None = None, + tls_max_version: TLSVersion | None = None, **kwargs: Any, ): - super().__init__(*args, **kwargs) - self._ssl_method: int = method - self.tls_verbose_logging: bool = tls_verbose_logging + super().__init__(*args, **kwargs) # type: ignore[no-untyped-call] + self._ssl_method: int | None = method + self.tls_min_version: TLSVersion | None = tls_min_version + self.tls_max_version: TLSVersion | None = tls_max_version + self.tls_verbose_logging: bool = tls_verbose_logging # unused self.tls_ciphers: AcceptableCiphers if tls_ciphers: self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) else: self.tls_ciphers = DEFAULT_CIPHERS - if method_is_overridden(type(self), ScrapyClientContextFactory, "getContext"): - warnings.warn( - "Overriding ScrapyClientContextFactory.getContext() is deprecated and that method" - " will be removed in a future Scrapy version. Override creatorForNetloc() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + self._verify_certificates = verify_certificates @classmethod def from_crawler( cls, crawler: Crawler, - method: int = SSL.SSLv23_METHOD, # noqa: S503 + method: int | None = SSL.SSLv23_METHOD, # noqa: S503 *args: Any, **kwargs: Any, ) -> Self: @@ -83,41 +87,86 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + # DOWNLOADER_CLIENT_TLS_METHOD reading and handling should be also moved here + # when the deprecated load_context_factory_from_settings() is removed + tls_min_ver, tls_max_ver = _get_tls_version_limits( + crawler.settings, _TWISTED_VERSION_MAP.__getitem__ + ) + if tls_min_ver or tls_max_ver: + method = None + verify_certificates = crawler.settings.getbool("DOWNLOAD_VERIFY_CERTIFICATES") return cls( # type: ignore[misc] *args, method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, + tls_min_version=tls_min_ver, + tls_max_version=tls_max_ver, + verify_certificates=verify_certificates, **kwargs, ) - def getCertificateOptions(self) -> CertificateOptions: - # setting verify=True will require you to provide CAs - # to verify against; in other words: it's not that simple - return CertificateOptions( - verify=False, - method=self._ssl_method, - fixBrokenPeers=True, - acceptableCiphers=self.tls_ciphers, - ) + # should be removed together with ScrapyClientContextFactory + def getCertificateOptions(self) -> CertificateOptions: # pragma: no cover + return self._get_cert_options() - # kept for old-style HTTP/1.0 downloader context twisted calls, - # e.g. connectSSL() - def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: - ctx: SSL.Context = self.getCertificateOptions().getContext() - ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT - return ctx + def _get_cert_options(self) -> CertificateOptions: + return _ScrapyCertificateOptions(**self._get_cert_options_kwargs()) + + def _get_cert_options_kwargs(self) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "fixBrokenPeers": True, + "acceptableCiphers": self.tls_ciphers, + } + if self.tls_min_version or self.tls_max_version: + kwargs.update( + _get_cert_options_version_kwargs( + self.tls_min_version, self.tls_max_version + ) + ) + # when ScrapyClientContextFactory is removed self._ssl_method can just be None by default + elif self._ssl_method != SSL.SSLv23_METHOD: + kwargs["method"] = self._ssl_method + return kwargs + + # should be removed together with ScrapyClientContextFactory + def getContext( + self, hostname: Any = None, port: Any = None + ) -> SSL.Context: # pragma: no cover + return self._get_context() + + def _get_context(self) -> SSL.Context: + return self._get_cert_options().getContext() def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: - return ScrapyClientTLSOptions( - hostname.decode("ascii"), - self.getContext(), - verbose_logging=self.tls_verbose_logging, + if not self._verify_certificates: + # Our options class is needed to skip verification errors + if TWISTED_TLS_NEW_IMPL: + return _ScrapyClientTLSOptions26( + self._get_cert_options()._makeTLSConnection, + hostname.decode("ascii"), + ) + return _ScrapyClientTLSOptions( + hostname.decode("ascii"), # type: ignore[arg-type] + self._get_context(), # type: ignore[arg-type] + ) + # Otherwise use the normal Twisted function. + return optionsForClientTLS( # type: ignore[no-any-return] + hostname=hostname.decode("ascii"), + extraCertificateOptions=self._get_cert_options_kwargs(), ) +ScrapyClientContextFactory = create_deprecated_class( + "ScrapyClientContextFactory", + _ScrapyClientContextFactory, + subclass_warn_message="{old} is deprecated.", + instance_warn_message="{cls} is deprecated.", +) + + @implementer(IPolicyForHTTPS) -class BrowserLikeContextFactory(ScrapyClientContextFactory): +class BrowserLikeContextFactory(_ScrapyClientContextFactory): """ Twisted-recommended context factory for web clients. @@ -130,30 +179,54 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): :meth:`creatorForNetloc` is the same as :class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context factory allows setting the TLS/SSL method to use. - - The default OpenSSL method is ``TLS_METHOD`` (also called - ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ + def __init__(self, *args: Any, **kwargs: Any): + warnings.warn( + "BrowserLikeContextFactory is deprecated." + " You can set DOWNLOAD_VERIFY_CERTIFICATES=True to enable" + " certificate verification instead of using it.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + super().__init__(*args, **kwargs) + def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: - # trustRoot set to platformTrust() will use the platform's root CAs. - # - # This means that a website like https://www.cacert.org will be rejected - # by default, since CAcert.org CA certificate is seldom shipped. - return optionsForClientTLS( + return optionsForClientTLS( # type: ignore[no-any-return] hostname=hostname.decode("ascii"), - trustRoot=platformTrust(), - extraCertificateOptions={"method": self._ssl_method}, + extraCertificateOptions=self._get_cert_options_kwargs(), ) @implementer(IPolicyForHTTPS) -class AcceptableProtocolsContextFactory: +class _AcceptableProtocolsContextFactory: """Context factory to used to override the acceptable protocols - to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN - negotiation. + to set up the :class:`OpenSSL.SSL.Context` for doing ALPN negotiation. + It's a private class for :class:`~.H2DownloadHandler`. + + This class wraps ``creatorForNetloc()`` of another factory class, setting + the acceptable protocols on the :class:`.ClientTLSOptions` instance + returned by it. It's only needed because we support custom factories via + :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`. + + It's a no-op on Twisted 26.4.0+, though using it with custom + factories on those Twisted versions may be not enough for HTTP/2 support. """ + # Something needs to call set_alpn_protos() for ALPN to work. + # + # Twisted < 26.4.0 does it in OpenSSLCertificateOptions._makeContext() + # (requires passing acceptableProtocols from the factory to + # OpenSSLCertificateOptions) and in TLSMemoryBIOFactory._createConnection() + # based on H2ClientFactory.acceptableProtocols (too late, it seems). + # + # Newer Twisted does it in OpenSSLCertificateOptions._makeContext() as + # well, and in OpenSSLCertificateOptions._makeTLSConnection() based on + # H2ClientFactory.acceptableProtocols (which now works). + # + # When we drop DOWNLOADER_CLIENTCONTEXTFACTORY it looks like we can replace + # all of this with _ScrapyClientContextFactory.acceptableProtocols. + def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]): verifyObject(IPolicyForHTTPS, context_factory) self._wrapped_context_factory: Any = context_factory @@ -163,35 +236,77 @@ class AcceptableProtocolsContextFactory: options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc( hostname, port ) - _setAcceptableProtocols(options._ctx, self._acceptable_protocols) + if not TWISTED_TLS_NEW_IMPL: + from twisted.internet._sslverify import ( # type: ignore[attr-defined] # noqa: PLC0415 # pylint: disable=no-name-in-module + _setAcceptableProtocols, + ) + + _setAcceptableProtocols(options._ctx, self._acceptable_protocols) # type: ignore[attr-defined] return options +AcceptableProtocolsContextFactory = create_deprecated_class( + "AcceptableProtocolsContextFactory", + _AcceptableProtocolsContextFactory, + subclass_warn_message="{old} is deprecated.", + instance_warn_message="{cls} is deprecated.", +) + + +class _ScrapyCertificateOptions(CertificateOptions): + """A wrapper needed to add flags to the SSL context before it's used.""" + + def _makeContext(self, skipCiphers: bool = False) -> SSL.Context: + if TWISTED_TLS_NEW_IMPL: + ctx = super()._makeContext(skipCiphers) + else: + ctx = super()._makeContext() + ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT + return ctx + + +def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS: + """Create an instance of :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`. + + Also passes values of other relevant settings to the factory class. + """ + tls_method_setting: str = crawler.settings["DOWNLOADER_CLIENT_TLS_METHOD"] + if tls_method_setting != "TLS": + warnings.warn( + "Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is" + " deprecated, please use DOWNLOAD_TLS_MIN_VERSION and/or" + " DOWNLOAD_TLS_MAX_VERSION instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + tls_method = _openssl_methods[tls_method_setting] + if crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL": + context_factory_cls = _ScrapyClientContextFactory + else: # pragma: no cover + warnings.warn( + "The 'DOWNLOADER_CLIENTCONTEXTFACTORY' setting is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + context_factory_cls = load_object( + crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] + ) + return cast( + "IPolicyForHTTPS", + build_from_crawler( + context_factory_cls, + crawler, + method=tls_method, + ), + ) + + def load_context_factory_from_settings( settings: BaseSettings, crawler: Crawler -) -> IPolicyForHTTPS: - ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")] - context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]) - # try method-aware context factory - try: - context_factory = build_from_crawler( - context_factory_cls, - crawler, - method=ssl_method, - ) - except TypeError: - # use context factory defaults - context_factory = build_from_crawler( - context_factory_cls, - crawler, - ) - msg = ( - f"{settings['DOWNLOADER_CLIENTCONTEXTFACTORY']} does not accept " - "a `method` argument (type OpenSSL.SSL method, e.g. " - "OpenSSL.SSL.SSLv23_METHOD) and/or a `tls_verbose_logging` " - "argument and/or a `tls_ciphers` argument. Please, upgrade your " - "context factory class to handle them or ignore them." - ) - warnings.warn(msg) - - return context_factory +) -> IPolicyForHTTPS: # pragma: no cover + warnings.warn( + "load_context_factory_from_settings() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _load_context_factory_from_settings(crawler) diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.py new file mode 100644 index 000000000..83d130462 --- /dev/null +++ b/scrapy/core/downloader/handlers/_base_http.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from abc import ABC +from typing import TYPE_CHECKING + +from .base import BaseDownloadHandler + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + +class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): + """Base class for built-in HTTP download handlers.""" + + def __init__(self, crawler: Crawler): + super().__init__(crawler) + self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE") + self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE") + self._fail_on_dataloss: bool = crawler.settings.getbool( + "DOWNLOAD_FAIL_ON_DATALOSS" + ) + self._tls_verbose_logging: bool = crawler.settings.getbool( + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" + ) + self._fail_on_dataloss_warned: bool = False diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py new file mode 100644 index 000000000..5a669c732 --- /dev/null +++ b/scrapy/core/downloader/handlers/_base_streaming.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +import base64 +import logging +import time +from abc import ABC, abstractmethod +from io import BytesIO +from typing import TYPE_CHECKING, Any, ClassVar, Generic, NoReturn, TypedDict, TypeVar +from urllib.parse import quote, urlsplit + +from scrapy import Request, signals +from scrapy.exceptions import ( + DownloadCancelledError, + NotConfigured, + ResponseDataLossError, +) +from scrapy.utils._download_handlers import ( + check_stop_download, + get_dataloss_msg, + get_maxsize_msg, + get_warnsize_msg, + make_response, + normalize_bind_address, +) +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.url import add_http_if_no_scheme + +from ._base_http import BaseHttpDownloadHandler + +if TYPE_CHECKING: + from collections.abc import AsyncIterable + from contextlib import AbstractAsyncContextManager + from ipaddress import IPv4Address, IPv6Address + + from _typeshed import SizedBuffer + + # typing.NotRequired requires Python 3.11 + from typing_extensions import NotRequired + + from scrapy.crawler import Crawler + from scrapy.http import Headers, Response + + +logger = logging.getLogger(__name__) + +_ResponseT = TypeVar("_ResponseT") + + +class _BaseResponseArgs(TypedDict): + status: int + url: str + headers: Headers + certificate: NotRequired[Any] + ip_address: NotRequired[IPv4Address | IPv6Address | None] + protocol: str | None + + +class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_ResponseT]): + """A base class for HTTP download handlers that follow the streaming logic flow.""" + + _DEFAULT_CONNECT_TIMEOUT: ClassVar[float] = 10 + experimental: ClassVar[bool] = False + requires_asyncio: ClassVar[bool] = True + # require subclasses to disable proxies explicitly with an explanation + supports_proxies: ClassVar[bool] = True + supports_per_request_bindaddress: ClassVar[bool] = False + + def __init__(self, crawler: Crawler): + if self.requires_asyncio and not is_asyncio_available(): # pragma: no cover + raise NotConfigured( + f"{type(self).__name__} requires the asyncio support. Make" + f" sure that you have either enabled the asyncio Twisted" + f" reactor in the TWISTED_REACTOR setting or disabled the" + f" TWISTED_REACTOR_ENABLED setting. See the asyncio documentation" + f" of Scrapy for more information." + ) + self._check_deps_installed() + super().__init__(crawler) + if self.experimental: + logger.warning( + f"{type(self).__name__} is experimental and is not recommended for production use." + ) + self._bind_address = normalize_bind_address( + crawler.settings.get("DOWNLOAD_BIND_ADDRESS") + ) + self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") + # these are useful for many handlers but used in different ways by them + self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS") + self._pool_size_per_host: int = crawler.settings.getint( + "CONCURRENT_REQUESTS_PER_DOMAIN" + ) + + @staticmethod + @abstractmethod + def _check_deps_installed() -> None: + """Raise NotConfigured if the required deps are not installed.""" + raise NotImplementedError + + @abstractmethod + def _make_request( + self, request: Request, timeout: float + ) -> AbstractAsyncContextManager[_ResponseT]: + """Return an async context manager yielding the library-specific response. + + Exceptions raised by the library should be reraised as Scrapy-specific ones. + """ + raise NotImplementedError + + @staticmethod + @abstractmethod + def _extract_headers(response: _ResponseT) -> Headers: + """Convert library-specific response headers to a + :class:`~scrapy.http.headers.Headers` object.""" + raise NotImplementedError + + @staticmethod + @abstractmethod + def _build_base_response_args( + response: _ResponseT, request: Request, headers: Headers + ) -> _BaseResponseArgs: + """Build kwargs for :func:`scrapy.utils._download_handlers.make_response`.""" + raise NotImplementedError + + @staticmethod + @abstractmethod + def _iter_body_chunks(response: _ResponseT) -> AsyncIterable[SizedBuffer]: + """Return an async iterable yielding body chunks from the response.""" + raise NotImplementedError + + @staticmethod + @abstractmethod + def _is_dataloss_exception(exc: Exception) -> bool: + """Return True if ``exc`` represents dataloss.""" + raise NotImplementedError + + def _log_tls_info(self, response: _ResponseT, request: Request) -> None: + """Log TLS connection details, if possible.""" + + async def download_request(self, request: Request) -> Response: + if not self.supports_proxies and request.meta.get("proxy"): + raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.") + if not self.supports_per_request_bindaddress and request.meta.get( + "bindaddress" + ): + logger.error( + f"The 'bindaddress' request meta key is not supported by" + f" {type(self).__name__} and will be ignored." + ) + timeout: float = request.meta.get( + "download_timeout", self._DEFAULT_CONNECT_TIMEOUT + ) + start_time = time.monotonic() + async with self._make_request(request, timeout) as response: + request.meta["download_latency"] = time.monotonic() - start_time + return await self._read_response(response, request) + + async def _read_response(self, response: _ResponseT, request: Request) -> Response: + maxsize: int = request.meta.get("download_maxsize", self._default_maxsize) + warnsize: int = request.meta.get("download_warnsize", self._default_warnsize) + + headers = self._extract_headers(response) + content_length = headers.get("Content-Length") + expected_size = int(content_length) if content_length is not None else None + if maxsize and expected_size and expected_size > maxsize: + self._cancel_maxsize(expected_size, maxsize, request, expected=True) + + reached_warnsize = False + if warnsize and expected_size and expected_size > warnsize: + reached_warnsize = True + logger.warning( + get_warnsize_msg(expected_size, warnsize, request, expected=True) + ) + + make_response_base_args = self._build_base_response_args( + response, request, headers + ) + + if self._tls_verbose_logging: + self._log_tls_info(response, request) + + if stop_download := check_stop_download( + signals.headers_received, + self.crawler, + request, + headers=headers, + body_length=expected_size, + ): + return make_response( + **make_response_base_args, + stop_download=stop_download, + ) + + response_body = BytesIO() + bytes_received = 0 + try: + async for chunk in self._iter_body_chunks(response): + response_body.write(chunk) + bytes_received += len(chunk) + + if stop_download := check_stop_download( + signals.bytes_received, self.crawler, request, data=chunk + ): + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + stop_download=stop_download, + ) + + if maxsize and bytes_received > maxsize: + response_body.truncate(0) + self._cancel_maxsize( + bytes_received, maxsize, request, expected=False + ) + + if warnsize and bytes_received > warnsize and not reached_warnsize: + reached_warnsize = True + logger.warning( + get_warnsize_msg( + bytes_received, warnsize, request, expected=False + ) + ) + except Exception as e: + if not self._is_dataloss_exception(e): + raise + fail_on_dataloss: bool = request.meta.get( + "download_fail_on_dataloss", self._fail_on_dataloss + ) + if not fail_on_dataloss: + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + flags=["dataloss"], + ) + if not self._fail_on_dataloss_warned: + logger.warning(get_dataloss_msg(request.url)) + self._fail_on_dataloss_warned = True + raise ResponseDataLossError(str(e)) from e + + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + ) + + @staticmethod + def _request_headers(request: Request) -> Headers: + """Get a prepared copy of the request headers. + + This removes the Proxy-Authorization header. + """ + headers = request.headers.copy() + headers.pop(b"Proxy-Authorization", None) + return headers + + def _get_bind_address_host(self) -> str | None: + """Return the host portion of the bind address. + + Needed for handlers that don't support the bind port. + """ + if self._bind_address is None: + return None + host, port = self._bind_address + if port != 0: + logger.warning( + "DOWNLOAD_BIND_ADDRESS specifies a port (%s), but %s does not " + "support binding to a specific local port. Ignoring the port " + "and binding only to %r.", + port, + type(self).__name__, + host, + ) + return host + + @staticmethod + def _cancel_maxsize( + size: int, limit: int, request: Request, *, expected: bool + ) -> NoReturn: + warning_msg = get_maxsize_msg(size, limit, request, expected=expected) + logger.warning(warning_msg) + raise DownloadCancelledError(warning_msg) + + @staticmethod + def _extract_proxy(request: Request) -> tuple[str | None, str | None]: + """Return a tuple of the proxy URL with a scheme and the value of the + Proxy-Authorization header. + + This is useful for handlers that take the proxy headers separately. + """ + proxy: str | None = request.meta.get("proxy") + if not proxy: + return None, None + proxy = add_http_if_no_scheme(proxy) + auth_header: bytes | None = request.headers.get(b"Proxy-Authorization") + return proxy, auth_header.decode("ascii") if auth_header else None + + def _extract_proxy_url_with_creds(self, request: Request) -> str | None: + """Return the proxy URL with the userinfo added based on the + Proxy-Authorization header. + + This is useful for handlers that cannot take the proxy headers + separately. + """ + proxy_url, auth_header = self._extract_proxy(request) + if proxy_url is None or auth_header is None: + return proxy_url + scheme, token = auth_header.split(" ", 1) + if scheme != "Basic": + raise ValueError( + f"Expected Basic auth in Proxy-Authorization, got {scheme}" + ) + user, password = ( + base64.b64decode(token).decode(self._proxy_auth_encoding).split(":", 1) + ) + parts = urlsplit(proxy_url) + netloc = f"{quote(user)}:{quote(password)}@{parts.netloc}" + return parts._replace(netloc=netloc).geturl() diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 4a9617506..9b54b44d8 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -3,104 +3,166 @@ from __future__ import annotations import ipaddress -import logging import ssl -from http.cookiejar import Cookie, CookieJar -from io import BytesIO -from typing import TYPE_CHECKING, Any, NoReturn, TypedDict +from contextlib import asynccontextmanager +from socket import gaierror +from typing import TYPE_CHECKING, ClassVar -import httpx - -from scrapy import Request, signals from scrapy.exceptions import ( CannotResolveHostError, - DownloadCancelledError, DownloadConnectionRefusedError, DownloadFailedError, DownloadTimeoutError, NotConfigured, - ResponseDataLossError, UnsupportedURLSchemeError, ) -from scrapy.http import Headers, Response -from scrapy.utils._download_handlers import ( - BaseHttpDownloadHandler, - check_stop_download, - get_dataloss_msg, - get_maxsize_msg, - get_warnsize_msg, - make_response, +from scrapy.http import Headers +from scrapy.utils._download_handlers import NullCookieJar +from scrapy.utils.python import _iter_exc_causes +from scrapy.utils.ssl import ( + _log_sslobj_debug_info, + _make_insecure_ssl_ctx, + _make_ssl_context, ) -from scrapy.utils.asyncio import is_asyncio_available -from scrapy.utils.ssl import _log_sslobj_debug_info, _make_ssl_context + +from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgs if TYPE_CHECKING: - from contextlib import AbstractAsyncContextManager - from http.client import HTTPResponse - from ipaddress import IPv4Address, IPv6Address - from urllib.request import Request as ULRequest + from collections.abc import AsyncIterator from httpcore import AsyncNetworkStream + from scrapy import Request from scrapy.crawler import Crawler -logger = logging.getLogger(__name__) +HAS_SOCKS = HAS_HTTP2 = False +try: + import httpx +except ImportError: + httpx = None # type: ignore[assignment] +else: + # a small hack to avoid importing these optional extras unconditionally -class _BaseResponseArgs(TypedDict): - status: int - url: str - headers: Headers - ip_address: IPv4Address | IPv6Address - protocol: str + DOWNLOAD_FAILED_EXCEPTIONS: tuple[type[BaseException], ...] = ( + httpx.RequestError, + httpx.InvalidURL, + ) + try: + import h2.exceptions -# workaround for (and from) https://github.com/encode/httpx/issues/2992 -class _NullCookieJar(CookieJar): # pragma: no cover - """A CookieJar that rejects all cookies.""" - - def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None: + HAS_HTTP2 = True + DOWNLOAD_FAILED_EXCEPTIONS += (h2.exceptions.InvalidBodyLengthError,) + except ImportError: # pragma: no cover pass - def set_cookie(self, cookie: Cookie) -> None: + try: + import socksio.exceptions + + HAS_SOCKS = True + DOWNLOAD_FAILED_EXCEPTIONS += (socksio.exceptions.ProtocolError,) + except ImportError: # pragma: no cover pass -class HttpxDownloadHandler(BaseHttpDownloadHandler): - _DEFAULT_CONNECT_TIMEOUT = 10 +if TYPE_CHECKING: + _Base = BaseStreamingDownloadHandler[httpx.Response] +else: + _Base = BaseStreamingDownloadHandler + + +class HttpxDownloadHandler(_Base): + experimental: ClassVar[bool] = True def __init__(self, crawler: Crawler): - # we don't run extra-deps tests with the non-asyncio reactor - if not is_asyncio_available(): # pragma: no cover - raise NotConfigured( - f"{type(self).__name__} requires the asyncio support. Make" - f" sure that you have either enabled the asyncio Twisted" - f" reactor in the TWISTED_REACTOR setting or disabled the" - f" TWISTED_ENABLED setting. See the asyncio documentation" - f" of Scrapy for more information." - ) super().__init__(crawler) - logger.warning( - "HttpxDownloadHandler is experimental and is not recommented for production use." + self._verify_certificates: bool = crawler.settings.getbool( + "DOWNLOAD_VERIFY_CERTIFICATES" ) - self._tls_verbose_logging: bool = self.crawler.settings.getbool( - "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" - ) - self._client = httpx.AsyncClient( - verify=_make_ssl_context(crawler.settings), cookies=_NullCookieJar() + self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED") + if self._enable_h2 and not HAS_HTTP2: # pragma: no cover + raise NotConfigured( + f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed." + ) + self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings) + self._bind_host: str | None = self._get_bind_address_host() + self._limits: httpx.Limits = httpx.Limits( + # hard limit on simultaneous connections + max_connections=self._pool_size_total, + # total number of idle connections in the pool (extra ones are closed) + max_keepalive_connections=self._pool_size_total, ) - async def download_request(self, request: Request) -> Response: - self._warn_unsupported_meta(request.meta) + self._default_client: httpx.AsyncClient = self._make_client() + # httpx doesn't support per-request proxies: https://github.com/encode/httpx/discussions/3183, + # so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed. + self._proxy_clients: dict[str, httpx.AsyncClient] = {} - timeout: float = request.meta.get( - "download_timeout", self._DEFAULT_CONNECT_TIMEOUT + @staticmethod + def _check_deps_installed() -> None: + if httpx is None: # pragma: no cover + raise NotConfigured( + "HttpxDownloadHandler requires the httpx library to be installed." + ) + + def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient: + if proxy_url: + if proxy_url.startswith("https:") and not self._verify_certificates: + proxy_ssl_context = _make_insecure_ssl_ctx() + else: + proxy_ssl_context = None + proxy = httpx.Proxy(proxy_url, ssl_context=proxy_ssl_context) + else: + proxy = None + + client = httpx.AsyncClient( + cookies=NullCookieJar(), + transport=httpx.AsyncHTTPTransport( + verify=self._ssl_context, + local_address=self._bind_host, + http2=self._enable_h2, + limits=self._limits, + trust_env=False, + proxy=proxy, + ), ) + # https://github.com/encode/httpx/discussions/1566 + for header_name in ("accept", "accept-encoding", "user-agent"): + client.headers.pop(header_name, None) + return client + + def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: + if proxy_url is None: + return self._default_client + if cached := self._proxy_clients.get(proxy_url): + return cached + client = self._make_client(proxy_url) + self._proxy_clients[proxy_url] = client + return client + + @asynccontextmanager + async def _make_request( + self, request: Request, timeout: float + ) -> AsyncIterator[httpx.Response]: + proxy = self._extract_proxy_url_with_creds(request) + if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover + raise ValueError( + f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed." + ) + client = self._get_client(proxy) + headers = self._request_headers(request).to_tuple_list() try: - async with self._get_httpx_response(request, timeout) as httpx_response: - return await self._read_response(httpx_response, request) + async with client.stream( + request.method, + request.url, + content=request.body, + headers=headers, + timeout=timeout, + ) as response: + yield response except httpx.TimeoutException as e: raise DownloadTimeoutError( f"Getting {request.url} took longer than {timeout} seconds." @@ -108,162 +170,58 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler): except httpx.UnsupportedProtocol as e: raise UnsupportedURLSchemeError(str(e)) from e except httpx.ConnectError as e: - if "Name or service not known" in str(e) or "getaddrinfo failed" in str(e): + if any(isinstance(c, gaierror) for c in _iter_exc_causes(e)): raise CannotResolveHostError(str(e)) from e raise DownloadConnectionRefusedError(str(e)) from e - except httpx.NetworkError as e: + except httpx.ProxyError as e: + raise DownloadConnectionRefusedError(str(e)) from e + except DOWNLOAD_FAILED_EXCEPTIONS as e: raise DownloadFailedError(str(e)) from e - except httpx.RemoteProtocolError as e: - raise DownloadFailedError(str(e)) from e - - def _warn_unsupported_meta(self, meta: dict[str, Any]) -> None: - if meta.get("bindaddress"): - # configurable only per-client: - # https://github.com/encode/httpx/issues/755#issuecomment-2746121794 - logger.error( - f"The 'bindaddress' request meta key is not supported by" - f" {type(self).__name__} and will be ignored." - ) - if meta.get("proxy"): - # configurable only per-client: - # https://github.com/encode/httpx/issues/486 - logger.error( - f"The 'proxy' request meta key is not supported by" - f" {type(self).__name__} and will be ignored." - ) - - def _get_httpx_response( - self, request: Request, timeout: float - ) -> AbstractAsyncContextManager[httpx.Response]: - return self._client.stream( - request.method, - request.url, - content=request.body, - headers=request.headers.to_tuple_list(), - timeout=timeout, - ) - - async def _read_response( - self, httpx_response: httpx.Response, request: Request - ) -> Response: - maxsize: int = request.meta.get("download_maxsize", self._default_maxsize) - warnsize: int = request.meta.get("download_warnsize", self._default_warnsize) - - content_length = httpx_response.headers.get("Content-Length") - expected_size = int(content_length) if content_length is not None else None - if maxsize and expected_size and expected_size > maxsize: - self._cancel_maxsize(expected_size, maxsize, request, expected=True) - - reached_warnsize = False - if warnsize and expected_size and expected_size > warnsize: - reached_warnsize = True - logger.warning( - get_warnsize_msg(expected_size, warnsize, request, expected=True) - ) - - headers = Headers(httpx_response.headers.multi_items()) - network_stream: AsyncNetworkStream = httpx_response.extensions["network_stream"] - - make_response_base_args: _BaseResponseArgs = { - "status": httpx_response.status_code, - "url": request.url, - "headers": headers, - "ip_address": self._get_server_ip(network_stream), - "protocol": httpx_response.http_version, - } - - self._log_tls_info(network_stream) - - if stop_download := check_stop_download( - signals.headers_received, - self.crawler, - request, - headers=headers, - body_length=expected_size, - ): - return make_response( - **make_response_base_args, - stop_download=stop_download, - ) - - response_body = BytesIO() - bytes_received = 0 - try: - async for chunk in httpx_response.aiter_raw(): - response_body.write(chunk) - bytes_received += len(chunk) - - if stop_download := check_stop_download( - signals.bytes_received, self.crawler, request, data=chunk - ): - return make_response( - **make_response_base_args, - body=response_body.getvalue(), - stop_download=stop_download, - ) - - if maxsize and bytes_received > maxsize: - response_body.truncate(0) - self._cancel_maxsize( - bytes_received, maxsize, request, expected=False - ) - - if warnsize and bytes_received > warnsize and not reached_warnsize: - reached_warnsize = True - logger.warning( - get_warnsize_msg( - bytes_received, warnsize, request, expected=False - ) - ) - except httpx.RemoteProtocolError as e: - # special handling of the dataloss case - if ( - "peer closed connection without sending complete message body" - not in str(e) - ): - raise - fail_on_dataloss: bool = request.meta.get( - "download_fail_on_dataloss", self._fail_on_dataloss - ) - if not fail_on_dataloss: - return make_response( - **make_response_base_args, - body=response_body.getvalue(), - flags=["dataloss"], - ) - self._log_dataloss_warning(request.url) - raise ResponseDataLossError(str(e)) from e - - return make_response( - **make_response_base_args, - body=response_body.getvalue(), - ) @staticmethod - def _get_server_ip(network_stream: AsyncNetworkStream) -> IPv4Address | IPv6Address: - extra_server_addr = network_stream.get_extra_info("server_addr") - return ipaddress.ip_address(extra_server_addr[0]) + def _extract_headers(response: httpx.Response) -> Headers: + return Headers(response.headers.multi_items()) - def _log_tls_info(self, network_stream: AsyncNetworkStream) -> None: - if not self._tls_verbose_logging: - return + @staticmethod + def _build_base_response_args( + response: httpx.Response, + request: Request, + headers: Headers, + ) -> _BaseResponseArgs: + network_stream: AsyncNetworkStream = response.extensions["network_stream"] + server_addr = network_stream.get_extra_info("server_addr") + ip_address = ipaddress.ip_address(server_addr[0]) + ssl_object = network_stream.get_extra_info("ssl_object") + if isinstance(ssl_object, ssl.SSLObject): + cert = ssl_object.getpeercert(binary_form=True) + else: + cert = None + return { + "status": response.status_code, + "url": request.url, + "headers": headers, + "certificate": cert, + "ip_address": ip_address, + "protocol": response.http_version, + } + + @staticmethod + def _iter_body_chunks(response: httpx.Response) -> AsyncIterator[bytes]: + return response.aiter_raw() + + @staticmethod + def _is_dataloss_exception(exc: Exception) -> bool: + return isinstance( + exc, httpx.RemoteProtocolError + ) and "peer closed connection without sending complete message body" in str(exc) + + def _log_tls_info(self, response: httpx.Response, request: Request) -> None: + network_stream: AsyncNetworkStream = response.extensions["network_stream"] extra_ssl_object = network_stream.get_extra_info("ssl_object") if isinstance(extra_ssl_object, ssl.SSLObject): _log_sslobj_debug_info(extra_ssl_object) - def _log_dataloss_warning(self, url: str) -> None: - if self._fail_on_dataloss_warned: - return - logger.warning(get_dataloss_msg(url)) - self._fail_on_dataloss_warned = True - - @staticmethod - def _cancel_maxsize( - size: int, limit: int, request: Request, *, expected: bool - ) -> NoReturn: - warning_msg = get_maxsize_msg(size, limit, request, expected=expected) - logger.warning(warning_msg) - raise DownloadCancelledError(warning_msg) - async def close(self) -> None: - await self._client.aclose() + await self._default_client.aclose() + for client in self._proxy_clients.values(): + await client.aclose() diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index 84f241d88..d0f32216c 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from w3lib.url import parse_data_uri @@ -17,9 +17,8 @@ class DataURIDownloadHandler(BaseDownloadHandler): uri = parse_data_uri(request.url) respcls = responsetypes.from_mimetype(uri.media_type) - resp_kwargs: dict[str, Any] = {} if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text": charset = uri.media_type_parameters.get("charset") - resp_kwargs["encoding"] = charset + return respcls(url=request.url, body=uri.data, encoding=charset) - return respcls(url=request.url, body=uri.data, **resp_kwargs) + return respcls(url=request.url, body=uri.data) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index a59aa722b..f080cce48 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -7,6 +7,7 @@ from w3lib.url import file_uri_to_path from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.responsetypes import responsetypes +from scrapy.utils.asyncio import run_in_thread if TYPE_CHECKING: from scrapy import Request @@ -16,6 +17,6 @@ if TYPE_CHECKING: class FileDownloadHandler(BaseDownloadHandler): async def download_request(self, request: Request) -> Response: filepath = file_uri_to_path(request.url) - body = Path(filepath).read_bytes() # noqa: ASYNC240 + body = await run_in_thread(Path(filepath).read_bytes) respcls = responsetypes.from_args(filename=filepath, body=body) return respcls(url=request.url, body=body) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index f261e8e7f..6258067c1 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -33,12 +33,13 @@ from __future__ import annotations import re from io import BytesIO from pathlib import Path -from typing import TYPE_CHECKING, BinaryIO +from typing import TYPE_CHECKING, BinaryIO, ClassVar from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol from scrapy.core.downloader.handlers.base import BaseDownloadHandler +from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.defer import maybe_deferred_to_future @@ -78,12 +79,14 @@ _CODE_RE = re.compile(r"\d+") class FTPDownloadHandler(BaseDownloadHandler): - CODE_MAPPING: dict[str, int] = { + CODE_MAPPING: ClassVar[dict[str, int]] = { "550": 404, "default": 503, } def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): + raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") super().__init__(crawler) self.default_user = crawler.settings["FTP_USER"] self.default_password = crawler.settings["FTP_PASSWORD"] diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index a7b592d00..dc1b1e375 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,7 +1,6 @@ # pragma: no file cover import warnings -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import ( HTTP11DownloadHandler as HTTPDownloadHandler, ) @@ -16,6 +15,5 @@ warnings.warn( ) __all__ = [ - "HTTP10DownloadHandler", "HTTPDownloadHandler", ] diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py deleted file mode 100644 index af3588019..000000000 --- a/scrapy/core/downloader/handlers/http10.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Download handlers for http and https schemes""" - -from __future__ import annotations - -import warnings -from typing import TYPE_CHECKING - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.defer import maybe_deferred_to_future -from scrapy.utils.misc import build_from_crawler, load_object -from scrapy.utils.python import to_unicode - -if TYPE_CHECKING: - from twisted.internet.interfaces import IConnector - - # typing.Self requires Python 3.11 - from typing_extensions import Self - - from scrapy import Request - from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory - from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory - from scrapy.crawler import Crawler - from scrapy.http import Response - from scrapy.settings import BaseSettings - - -class HTTP10DownloadHandler: - lazy = False - - def __init__(self, settings: BaseSettings, crawler: Crawler): - warnings.warn( - "HTTP10DownloadHandler is deprecated and will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object( - settings["DOWNLOADER_HTTPCLIENTFACTORY"] - ) - self.ClientContextFactory: type[ScrapyClientContextFactory] = load_object( - settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] - ) - self._settings: BaseSettings = settings - self._crawler: Crawler = crawler - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings, crawler) - - async def download_request(self, request: Request) -> Response: - factory = self.HTTPClientFactory(request) - self._connect(factory) - return await maybe_deferred_to_future(factory.deferred) - - def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector: - from twisted.internet import reactor - - host, port = to_unicode(factory.host), factory.port - if factory.scheme == b"https": - client_context_factory = build_from_crawler( - self.ClientContextFactory, - self._crawler, - ) - return reactor.connectSSL(host, port, factory, client_context_factory) - return reactor.connectTCP(host, port, factory) - - async def close(self) -> None: - pass diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 2fd3bd988..85a5ae0d7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -6,8 +6,9 @@ import ipaddress import logging import re from contextlib import suppress +from functools import partial from io import BytesIO -from time import time +from time import monotonic from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast from urllib.parse import urldefrag, urlparse @@ -30,29 +31,33 @@ from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IRe from zope.interface import implementer from scrapy import Request, signals -from scrapy.core.downloader.contextfactory import load_context_factory_from_settings +from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings from scrapy.exceptions import ( DownloadCancelledError, DownloadTimeoutError, + NotConfigured, ResponseDataLossError, StopDownload, ) from scrapy.http import Headers, Response from scrapy.utils._download_handlers import ( - BaseHttpDownloadHandler, check_stop_download, get_dataloss_msg, get_maxsize_msg, get_warnsize_msg, make_response, + normalize_bind_address, wrap_twisted_exceptions, ) from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.ssl import _log_ssl_conn_debug_info from scrapy.utils.url import add_http_if_no_scheme +from ._base_http import BaseHttpDownloadHandler + if TYPE_CHECKING: from twisted.internet.base import ReactorBase from twisted.internet.interfaces import IConsumer @@ -79,6 +84,8 @@ class _ResultT(TypedDict): class HTTP11DownloadHandler(BaseHttpDownloadHandler): def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): + raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") super().__init__(crawler) self._crawler = crawler @@ -90,9 +97,10 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): ) self._pool._factory.noisy = False - self._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings( - crawler.settings, crawler + self._contextFactory: IPolicyForHTTPS = _load_context_factory_from_settings( + crawler ) + self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS") self._disconnect_timeout: int = 1 async def download_request(self, request: Request) -> Response: @@ -104,8 +112,9 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): "download_warnsize", "DOWNLOAD_WARNSIZE" ) - agent = ScrapyAgent( + agent = _ScrapyAgent( contextFactory=self._contextFactory, + bindAddress=self._bind_address, pool=self._pool, maxsize=getattr( self._crawler.spider, "download_maxsize", self._default_maxsize @@ -115,6 +124,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): ), fail_on_dataloss=self._fail_on_dataloss, crawler=self._crawler, + tls_verbose_logging=self._tls_verbose_logging, ) try: with wrap_twisted_exceptions(): @@ -139,7 +149,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): # issue a callback after `_disconnect_timeout` seconds. # # See also https://github.com/scrapy/scrapy/issues/2653 - delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) + delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, ()) try: await maybe_deferred_to_future(d) @@ -152,7 +162,7 @@ class TunnelError(Exception): """An HTTP CONNECT tunnel could not be established by the proxy.""" -class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): +class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """An endpoint that tunnels through proxies to allow HTTPS downloads. To accomplish that, this endpoint sends an HTTP CONNECT to the proxy. The HTTP CONNECT is always sent when using this endpoint, I think this could @@ -188,7 +198,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def requestTunnel(self, protocol: Protocol) -> Protocol: """Asks the proxy to open a tunnel.""" assert protocol.transport - tunnelReq = tunnel_request_data( + tunnelReq = _tunnel_request_data( self._tunneledHost, self._tunneledPort, self._proxyAuthHeader ) protocol.transport.write(tunnelReq) @@ -212,11 +222,12 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): if b"\r\n\r\n" not in self._connectBuffer: return self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign] - respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) + respm = _TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group("status")) == 200: # set proper Server Name Indication extension sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc] - self._tunneledHost, self._tunneledPort + self._tunneledHost, # type: ignore[arg-type] + self._tunneledPort, ) self._protocol.transport.startTLS(sslOptions, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) @@ -248,18 +259,18 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): return self._tunnelReadyDeferred -def tunnel_request_data( +def _tunnel_request_data( host: str, port: int, proxy_auth_header: bytes | None = None ) -> bytes: r""" Return binary content of a CONNECT request. >>> from scrapy.utils.python import to_unicode as s - >>> s(tunnel_request_data("example.com", 8080)) + >>> s(_tunnel_request_data("example.com", 8080)) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' - >>> s(tunnel_request_data("example.com", 8080, b"123")) + >>> s(_tunnel_request_data("example.com", 8080, b"123")) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n' - >>> s(tunnel_request_data(b"example.com", "8090")) + >>> s(_tunnel_request_data(b"example.com", "8090")) 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n' """ host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port)) @@ -271,7 +282,7 @@ def tunnel_request_data( return tunnel_req -class TunnelingAgent(Agent): +class _TunnelingAgent(Agent): """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS downloads. It may look strange that we have chosen to subclass Agent and not ProxyAgent but consider that after the tunnel is opened the proxy is @@ -286,15 +297,15 @@ class TunnelingAgent(Agent): proxyConf: tuple[str, int, bytes | None], contextFactory: IPolicyForHTTPS, connectTimeout: float | None = None, - bindAddress: bytes | None = None, + bindAddress: tuple[str, int] | None = None, pool: HTTPConnectionPool | None = None, ): - super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) + super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) # type: ignore[no-untyped-call] self._proxyConf: tuple[str, int, bytes | None] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory - def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: - return TunnelingTCP4ClientEndpoint( + def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint: + return _TunnelingTCP4ClientEndpoint( reactor=self._reactor, host=uri.host, port=uri.port, @@ -329,17 +340,19 @@ class TunnelingAgent(Agent): ) -class ScrapyProxyAgent(Agent): +class _ScrapyProxyAgent(Agent): def __init__( self, reactor: ReactorBase, proxyURI: bytes, + contextFactory: IPolicyForHTTPS, connectTimeout: float | None = None, - bindAddress: bytes | None = None, + bindAddress: tuple[str, int] | None = None, pool: HTTPConnectionPool | None = None, ): - super().__init__( + super().__init__( # type: ignore[no-untyped-call] reactor=reactor, + contextFactory=contextFactory, connectTimeout=connectTimeout, bindAddress=bindAddress, pool=pool, @@ -360,7 +373,7 @@ class ScrapyProxyAgent(Agent): # connecting to a single destination, the proxy: return self._requestWithEndpoint( key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port), - endpoint=self._getEndpoint(self._proxyURI), + endpoint=self._getEndpoint(self._proxyURI), # type: ignore[no-untyped-call] method=method, parsedURI=URI.fromBytes(uri), headers=headers, @@ -369,37 +382,36 @@ class ScrapyProxyAgent(Agent): ) -class ScrapyAgent: - _Agent = Agent - _ProxyAgent = ScrapyProxyAgent - _TunnelingAgent = TunnelingAgent - +class _ScrapyAgent: def __init__( self, *, contextFactory: IPolicyForHTTPS, connectTimeout: float = 10, - bindAddress: bytes | None = None, + bindAddress: str | tuple[str, int] | None = None, pool: HTTPConnectionPool | None = None, maxsize: int = 0, warnsize: int = 0, fail_on_dataloss: bool = True, crawler: Crawler, + tls_verbose_logging: bool = False, ): self._contextFactory: IPolicyForHTTPS = contextFactory self._connectTimeout: float = connectTimeout - self._bindAddress: bytes | None = bindAddress + self._bindAddress: str | tuple[str, int] | None = bindAddress self._pool: HTTPConnectionPool | None = pool self._maxsize: int = maxsize self._warnsize: int = warnsize self._fail_on_dataloss: bool = fail_on_dataloss self._txresponse: TxResponse | None = None self._crawler: Crawler = crawler + self._tls_verbose_logging: bool = tls_verbose_logging def _get_agent(self, request: Request, timeout: float) -> Agent: from twisted.internet import reactor bindaddress = request.meta.get("bindaddress") or self._bindAddress + bindaddress = normalize_bind_address(bindaddress) proxy = request.meta.get("proxy") if proxy: proxy = add_http_if_no_scheme(proxy) @@ -409,10 +421,14 @@ class ScrapyAgent: if not proxy_port: proxy_port = 443 if proxy_parsed.scheme == "https" else 80 if urlparse_cached(request).scheme == "https": + if proxy_parsed.scheme == "https": # pragma: no cover + raise NotImplementedError( + "HTTPS proxies for HTTPS destinations are not supported" + ) assert proxy_host is not None proxyAuth = request.headers.get(b"Proxy-Authorization", None) proxyConf = (proxy_host, proxy_port, proxyAuth) - return self._TunnelingAgent( + return _TunnelingAgent( reactor=reactor, proxyConf=proxyConf, contextFactory=self._contextFactory, @@ -420,15 +436,16 @@ class ScrapyAgent: bindAddress=bindaddress, pool=self._pool, ) - return self._ProxyAgent( + return _ScrapyProxyAgent( reactor=reactor, proxyURI=to_bytes(proxy, encoding="ascii"), + contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool, ) - return self._Agent( + return Agent( reactor=reactor, contextFactory=self._contextFactory, connectTimeout=timeout, @@ -446,10 +463,10 @@ class ScrapyAgent: url = urldefrag(request.url)[0] method = to_bytes(request.method) headers = TxHeaders(request.headers) - if isinstance(agent, self._TunnelingAgent): + if isinstance(agent, _TunnelingAgent): headers.removeHeader(b"Proxy-Authorization") bodyproducer = _RequestBodyProducer(request.body) if request.body else None - start_time = time() + start_time = monotonic() d: Deferred[IResponse] = agent.request( method, to_bytes(url, encoding="ascii"), @@ -478,7 +495,7 @@ class ScrapyAgent: raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.") def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T: - request.meta["download_latency"] = time() - start_time + request.meta["download_latency"] = monotonic() - start_time return result @staticmethod @@ -536,11 +553,7 @@ class ScrapyAgent: get_warnsize_msg(expected_size, warnsize, request, expected=True) ) - def _cancel(_: Any) -> None: - # Abort connection immediately. - txresponse._transport._producer.abortConnection() - - d: Deferred[_ResultT] = Deferred(_cancel) + d: Deferred[_ResultT] = Deferred(partial(self._cancel, txresponse=txresponse)) txresponse.deliverBody( _ResponseReader( finished=d, @@ -550,6 +563,7 @@ class ScrapyAgent: warnsize=warnsize, fail_on_dataloss=fail_on_dataloss, crawler=self._crawler, + tls_verbose_logging=self._tls_verbose_logging, ) ) @@ -558,6 +572,11 @@ class ScrapyAgent: return d + @staticmethod + def _cancel(_: Any, txresponse: TxResponse) -> None: + # Abort connection immediately. + txresponse._transport._producer.abortConnection() + def _cb_bodydone(self, result: _ResultT, url: str) -> Response: headers = self._headers_from_twisted_response(result["txresponse"]) try: @@ -605,6 +624,8 @@ class _ResponseReader(Protocol): warnsize: int, fail_on_dataloss: bool, crawler: Crawler, + *, + tls_verbose_logging: bool = False, ): self._finished: Deferred[_ResultT] = finished self._txresponse: TxResponse = txresponse @@ -618,6 +639,7 @@ class _ResponseReader(Protocol): self._certificate: ssl.Certificate | None = None self._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None self._crawler: Crawler = crawler + self._tls_verbose_logging: bool = tls_verbose_logging def _finish_response( self, flags: list[str] | None = None, stop_download: StopDownload | None = None @@ -646,17 +668,23 @@ class _ResponseReader(Protocol): self.transport._producer.getPeer().host ) - def dataReceived(self, bodyBytes: bytes) -> None: + if self._tls_verbose_logging: + connection = self.transport._producer.getHandle() + hostname = urlparse_cached(self._request).hostname + assert hostname is not None + _log_ssl_conn_debug_info(hostname, connection) + + def dataReceived(self, data: bytes) -> None: # This maybe called several times after cancel was called with buffered data. if self._finished.called: return assert self.transport - self._bodybuf.write(bodyBytes) - self._bytes_received += len(bodyBytes) + self._bodybuf.write(data) + self._bytes_received += len(data) if stop_download := check_stop_download( - signals.bytes_received, self._crawler, self._request, data=bodyBytes + signals.bytes_received, self._crawler, self._request, data=data ): self.transport.stopProducing() self.transport.loseConnection() diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 75fa0772d..f60c58d1b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,19 +1,23 @@ from __future__ import annotations -from time import time +from time import monotonic from typing import TYPE_CHECKING from urllib.parse import urldefrag -from twisted.web.client import URI - -from scrapy.core.downloader.contextfactory import load_context_factory_from_settings +from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings from scrapy.core.downloader.handlers.base import BaseDownloadHandler -from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent -from scrapy.exceptions import DownloadTimeoutError -from scrapy.utils._download_handlers import wrap_twisted_exceptions +from scrapy.core.http2.agent import H2Agent, H2ConnectionPool +from scrapy.exceptions import ( + DownloadTimeoutError, + NotConfigured, + UnsupportedURLSchemeError, +) +from scrapy.utils._download_handlers import ( + normalize_bind_address, + wrap_twisted_exceptions, +) from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes if TYPE_CHECKING: from twisted.internet.base import DelayedCall @@ -29,20 +33,26 @@ class H2DownloadHandler(BaseDownloadHandler): lazy = True def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): + raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") super().__init__(crawler) self._crawler = crawler from twisted.internet import reactor self._pool = H2ConnectionPool(reactor, crawler.settings) - self._context_factory = load_context_factory_from_settings( - crawler.settings, crawler - ) + self._context_factory = _load_context_factory_from_settings(crawler) + self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS") async def download_request(self, request: Request) -> Response: - agent = ScrapyH2Agent( + if urlparse_cached(request).scheme == "http": # pragma: no cover + raise UnsupportedURLSchemeError( + f"{type(self).__name__} doesn't support plain HTTP." + ) + agent = _ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, + bind_address=self._bind_address, crawler=self._crawler, ) assert self._crawler.spider @@ -55,16 +65,13 @@ class H2DownloadHandler(BaseDownloadHandler): self._pool.close_connections() -class ScrapyH2Agent: - _Agent = H2Agent - _ProxyAgent = ScrapyProxyH2Agent - +class _ScrapyH2Agent: def __init__( self, context_factory: IPolicyForHTTPS, pool: H2ConnectionPool, connect_timeout: int = 10, - bind_address: bytes | None = None, + bind_address: str | tuple[str, int] | None = None, crawler: Crawler | None = None, ) -> None: self._context_factory = context_factory @@ -76,24 +83,11 @@ class ScrapyH2Agent: def _get_agent(self, request: Request, timeout: float | None) -> H2Agent: from twisted.internet import reactor + if request.meta.get("proxy"): # pragma: no cover + raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.") bind_address = request.meta.get("bindaddress") or self._bind_address - proxy = request.meta.get("proxy") - if proxy: - if urlparse_cached(request).scheme == "https": - # ToDo - raise NotImplementedError( - "Tunneling via CONNECT method using HTTP/2.0 is not yet supported" - ) - return self._ProxyAgent( - reactor=reactor, - context_factory=self._context_factory, - proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")), - connect_timeout=timeout, - bind_address=bind_address, - pool=self._pool, - ) - - return self._Agent( + bind_address = normalize_bind_address(bind_address) + return H2Agent( reactor=reactor, context_factory=self._context_factory, connect_timeout=timeout, @@ -107,7 +101,7 @@ class ScrapyH2Agent: timeout = request.meta.get("download_timeout") or self._connect_timeout agent = self._get_agent(request, timeout) - start_time = time() + start_time = monotonic() d = agent.request(request, spider) d.addCallback(self._cb_latency, request, start_time) @@ -119,7 +113,7 @@ class ScrapyH2Agent: def _cb_latency( response: Response, request: Request, start_time: float ) -> Response: - request.meta["download_latency"] = time() - start_time + request.meta["download_latency"] = monotonic() - start_time return response @staticmethod diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index a601bc2aa..c6660ff48 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,15 +1,16 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, cast from scrapy.core.downloader.handlers.base import BaseDownloadHandler -from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import build_from_crawler +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: + from collections.abc import Mapping + from scrapy import Request from scrapy.crawler import Crawler from scrapy.http import Response @@ -40,7 +41,10 @@ class S3DownloadHandler(BaseDownloadHandler): ) ) - _http_handler = build_from_crawler(HTTP11DownloadHandler, crawler) + _http_handler: BaseDownloadHandler = build_from_crawler( + load_object(crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"]), + crawler, + ) self._download_http = _http_handler.download_request async def download_request(self, request: Request) -> Response: @@ -57,7 +61,7 @@ class S3DownloadHandler(BaseDownloadHandler): awsrequest = botocore.awsrequest.AWSRequest( method=request.method, url=f"{scheme}://s3.amazonaws.com/{bucket}{path}", - headers=request.headers.to_unicode_dict(), + headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()), data=request.body, ) assert self._signer diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 431579856..ab74e22a4 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -8,7 +8,7 @@ from __future__ import annotations import warnings from functools import wraps -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Request, Response @@ -36,7 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: - return build_component_list(settings.getwithbase("DOWNLOADER_MIDDLEWARES")) + return build_component_list( + settings.get_component_priority_dict_with_base("DOWNLOADER_MIDDLEWARES") + ) def _add_middleware(self, mw: Any) -> None: if hasattr(mw, "process_request"): @@ -73,87 +75,82 @@ class DownloaderMiddlewareManager(MiddlewareManager): download_func: Callable[[Request], Coroutine[Any, Any, Response]], request: Request, ) -> Response | Request: - async def process_request(request: Request) -> Response | Request: - for method in self.methods["process_request"]: - method = cast("Callable", method) - if method in self._mw_methods_requiring_spider: - response = await ensure_awaitable( - method(request=request, spider=self._spider), - _warn=global_object_name(method), - ) - else: - response = await ensure_awaitable( - method(request=request), _warn=global_object_name(method) - ) - if response is not None and not isinstance( - response, (Response, Request) - ): - raise _InvalidOutput( - f"Middleware {method.__qualname__} must return None, Response or " - f"Request, got {response.__class__.__name__}" - ) - if response: - return response - return await download_func(request) - - async def process_response(response: Response | Request) -> Response | Request: - if response is None: - raise TypeError("Received None in process_response") - if isinstance(response, Request): - return response - - for method in self.methods["process_response"]: - method = cast("Callable", method) - if method in self._mw_methods_requiring_spider: - response = await ensure_awaitable( - method(request=request, response=response, spider=self._spider), - _warn=global_object_name(method), - ) - else: - response = await ensure_awaitable( - method(request=request, response=response), - _warn=global_object_name(method), - ) - if not isinstance(response, (Response, Request)): - raise _InvalidOutput( - f"Middleware {method.__qualname__} must return Response or Request, " - f"got {type(response)}" - ) - if isinstance(response, Request): - return response - return response - - async def process_exception(exception: Exception) -> Response | Request: - for method in self.methods["process_exception"]: - method = cast("Callable", method) - if method in self._mw_methods_requiring_spider: - response = await ensure_awaitable( - method( - request=request, exception=exception, spider=self._spider - ), - _warn=global_object_name(method), - ) - else: - response = await ensure_awaitable( - method(request=request, exception=exception), - _warn=global_object_name(method), - ) - if response is not None and not isinstance( - response, (Response, Request) - ): - raise _InvalidOutput( - f"Middleware {method.__qualname__} must return None, Response or " - f"Request, got {type(response)}" - ) - if response: - return response - raise exception try: - result: Response | Request = await process_request(request) + result: Response | Request = await self._process_request( + request, download_func + ) except Exception as ex: await _defer_sleep_async() # either returns a request or response (which we pass to process_response()) # or reraises the exception - result = await process_exception(ex) - return await process_response(result) + result = await self._process_exception(ex, request) + return await self._process_response(result, request) + + def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any: + if method in self._mw_methods_requiring_spider: + kwargs["spider"] = self._spider + + return method(**kwargs) + + async def _process_request( + self, + request: Request, + download_func: Callable[[Request], Coroutine[Any, Any, Response]], + ) -> Response | Request: + for method in self.methods["process_request"]: + assert method is not None + response = await ensure_awaitable( + self._handle_mw_method(method, request=request), + _warn=global_object_name(method), + ) + if response is not None and not isinstance(response, (Response, Request)): + raise _InvalidOutput( + f"Middleware {method.__qualname__} must return None, Response or " + f"Request, got {response.__class__.__name__}" + ) + if response: + return response + return await download_func(request) + + async def _process_response( + self, response: Response | Request, request: Request + ) -> Response | Request: + if response is None: + raise TypeError("Received None in process_response") + if isinstance(response, Request): + return response + + for method in self.methods["process_response"]: + assert method is not None + response = await ensure_awaitable( + self._handle_mw_method(method, request=request, response=response), + _warn=global_object_name(method), + ) + + if not isinstance(response, (Response, Request)): + raise _InvalidOutput( + f"Middleware {method.__qualname__} must return Response or Request, " + f"got {type(response)}" + ) + if isinstance(response, Request): + return response + return response + + async def _process_exception( + self, exception: Exception, request: Request | Response + ) -> Response | Request: + for method in self.methods["process_exception"]: + assert method is not None + response = await ensure_awaitable( + self._handle_mw_method(method, request=request, exception=exception), + _warn=global_object_name(method), + ) + if response is not None and not isinstance(response, (Response, Request)): + raise _InvalidOutput( + f"Middleware {method.__qualname__} must return None, Response or " + f"Request, got {type(response)}" + ) + if response: + return response + raise exception diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 1ae66f614..3b903b39d 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,35 +1,74 @@ +from __future__ import annotations + import logging -from typing import Any +import warnings +from typing import TYPE_CHECKING, Any from OpenSSL import SSL +from service_identity import VerificationError from service_identity.exceptions import CertificateError -from twisted.internet._sslverify import ( - ClientTLSOptions, - VerificationError, - verifyHostname, +from service_identity.hazmat import ( + DNS_ID, + IPAddress_ID, + ServiceID, + verify_service_identity, ) -from twisted.internet.ssl import AcceptableCiphers +from service_identity.pyopenssl import ( + extract_patterns, + verify_hostname, + verify_ip_address, +) +from twisted.internet._sslverify import ClientTLSOptions +from twisted.internet.ssl import AcceptableCiphers, TLSVersion + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.deprecate import create_deprecated_class + +if TYPE_CHECKING: + from collections.abc import Callable + + from OpenSSL.crypto import X509 + from twisted.protocols.tls import TLSMemoryBIOProtocol -from scrapy.utils.ssl import get_temp_key_info, x509name_to_string logger = logging.getLogger(__name__) -METHOD_TLS = "TLS" -METHOD_TLSv10 = "TLSv1.0" -METHOD_TLSv11 = "TLSv1.1" -METHOD_TLSv12 = "TLSv1.2" - - -openssl_methods: dict[str, int] = { - METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only - METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only - METHOD_TLSv12: SSL.TLSv1_2_METHOD, # TLS 1.2 only +_openssl_methods: dict[str, int] = { + "TLS": SSL.SSLv23_METHOD, # protocol negotiation (recommended) + "TLSv1.0": SSL.TLSv1_METHOD, # TLS 1.0 only + "TLSv1.1": SSL.TLSv1_1_METHOD, # TLS 1.1 only + "TLSv1.2": SSL.TLSv1_2_METHOD, # TLS 1.2 only } -class ScrapyClientTLSOptions(ClientTLSOptions): +def __getattr__(name: str) -> Any: + deprecated = { + "METHOD_TLS": "TLS", + "METHOD_TLSv10": "TLSv1.0", + "METHOD_TLSv11": "TLSv1.1", + "METHOD_TLSv12": "TLSv1.2", + "openssl_methods": _openssl_methods, + } + if name in deprecated: + warnings.warn( + f"scrapy.core.downloader.tls.{name} is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deprecated[name] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +_TWISTED_VERSION_MAP: dict[str, TLSVersion] = { + "TLSv1.0": TLSVersion.TLSv1_0, + "TLSv1.1": TLSVersion.TLSv1_1, + "TLSv1.2": TLSVersion.TLSv1_2, + "TLSv1.3": TLSVersion.TLSv1_3, +} + + +class _ScrapyClientTLSOptions(ClientTLSOptions): """ SSL Client connection creator ignoring certificate verification errors (for genuinely invalid certificates or bugs in verification code). @@ -37,46 +76,29 @@ class ScrapyClientTLSOptions(ClientTLSOptions): Same as Twisted's private _sslverify.ClientTLSOptions, except that VerificationError, CertificateError and ValueError exceptions are caught, so that the connection is not closed, only - logging warnings. Also, HTTPS connection parameters logging is added. - """ + logging warnings. - def __init__(self, hostname: str, ctx: SSL.Context, verbose_logging: bool = False): - super().__init__(hostname, ctx) - self.verbose_logging: bool = verbose_logging + Instances of this class are returned from + :class:`._ScrapyClientContextFactory`. + + This class is used on Twisted older than 26.4.0. + """ def _identityVerifyingInfoCallback( self, connection: SSL.Connection, where: int, ret: Any ) -> None: - if where & SSL.SSL_CB_HANDSHAKE_START: - connection.set_tlsext_host_name(self._hostnameBytes) - elif where & SSL.SSL_CB_HANDSHAKE_DONE: - if self.verbose_logging: - logger.debug( - "SSL connection to %s using protocol %s, cipher %s", - self._hostnameASCII, - connection.get_protocol_version_name(), - connection.get_cipher_name(), - ) - server_cert = connection.get_peer_certificate() - if server_cert: - logger.debug( - 'SSL connection certificate: issuer "%s", subject "%s"', - x509name_to_string(server_cert.get_issuer()), - x509name_to_string(server_cert.get_subject()), - ) - key_info = get_temp_key_info(connection._ssl) - if key_info: - logger.debug("SSL temp key: %s", key_info) - + if where & SSL.SSL_CB_HANDSHAKE_DONE: try: - verifyHostname(connection, self._hostnameASCII) + if self._hostnameIsDnsName: + verify_hostname(connection, self._hostnameASCII) + else: + verify_ip_address(connection, self._hostnameASCII) except (CertificateError, VerificationError) as e: logger.warning( 'Remote certificate is not valid for hostname "%s"; %s', self._hostnameASCII, e, ) - except ValueError as e: logger.warning( "Ignoring error while verifying certificate " @@ -84,6 +106,77 @@ class ScrapyClientTLSOptions(ClientTLSOptions): self._hostnameASCII, e, ) + else: + super()._identityVerifyingInfoCallback(connection, where, ret) # type: ignore[misc] + + +ScrapyClientTLSOptions = create_deprecated_class( + "ScrapyClientTLSOptions", + _ScrapyClientTLSOptions, + subclass_warn_message="{old} is deprecated.", + instance_warn_message="{cls} is deprecated.", +) + + +class _ScrapyClientTLSOptions26(ClientTLSOptions): + """ + SSL Client connection creator ignoring certificate verification errors + (for genuinely invalid certificates or bugs in verification code). + + Same as Twisted's private _sslverify.ClientTLSOptions, + except that VerificationError, CertificateError and ValueError + exceptions are caught, so that the connection is not closed, only + logging warnings. + + Instances of this class are returned from + :class:`._ScrapyClientContextFactory`. + + This class is used on Twisted 26.4.0 and newer. + """ + + def clientConnectionForTLS( + self, tlsProtocol: TLSMemoryBIOProtocol + ) -> SSL.Connection: + """This method is needed to override the verify callback.""" + conn = super().clientConnectionForTLS(tlsProtocol) + callback = self._verifyCB(self._hostnameIsDnsName, self._hostnameASCII) + conn.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, callback) + return conn + + @staticmethod + def _verifyCB( + hostIsDNS: bool, hostnameASCII: str + ) -> Callable[[SSL.Connection, X509, int, int, int], bool]: + svcid: ServiceID = ( + DNS_ID(hostnameASCII) if hostIsDNS else IPAddress_ID(hostnameASCII) + ) + + def verifyCallback( + conn: SSL.Connection, cert: X509, err: int, depth: int, ok: int + ) -> bool: + if depth != 0: + # We are only verifying the leaf certificate. + return True + + try: + verify_service_identity(extract_patterns(cert), [svcid], []) + except (CertificateError, VerificationError) as e: + logger.warning( + 'Remote certificate is not valid for hostname "%s"; %s', + hostnameASCII, + e, + ) + except ValueError as e: + logger.warning( + "Ignoring error while verifying certificate " + 'from host "%s" (exception: %r)', + hostnameASCII, + e, + ) + + return True + + return verifyCallback DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString( diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py deleted file mode 100644 index 52d287245..000000000 --- a/scrapy/core/downloader/webclient.py +++ /dev/null @@ -1,239 +0,0 @@ -"""Deprecated HTTP/1.0 helper classes used by HTTP10DownloadHandler.""" - -from __future__ import annotations - -import warnings -from time import time -from typing import TYPE_CHECKING -from urllib.parse import urldefrag, urlparse, urlunparse - -from twisted.internet import defer -from twisted.internet.protocol import ClientFactory -from twisted.web.http import HTTPClient - -from scrapy.exceptions import DownloadTimeoutError, ScrapyDeprecationWarning -from scrapy.http import Headers, Response -from scrapy.responsetypes import responsetypes -from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes, to_unicode - -if TYPE_CHECKING: - from scrapy import Request - - -class ScrapyHTTPPageGetter(HTTPClient): - delimiter = b"\n" - - def __init__(self): - warnings.warn( - "ScrapyHTTPPageGetter is deprecated and will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - super().__init__() - - def connectionMade(self): - self.headers = Headers() # bucket for response headers - - # Method command - self.sendCommand(self.factory.method, self.factory.path) - # Headers - for key, values in self.factory.headers.items(): - for value in values: - self.sendHeader(key, value) - self.endHeaders() - # Body - if self.factory.body is not None: - self.transport.write(self.factory.body) - - def lineReceived(self, line): - return HTTPClient.lineReceived(self, line.rstrip()) - - def handleHeader(self, key, value): - self.headers.appendlist(key, value) - - def handleStatus(self, version, status, message): - self.factory.gotStatus(version, status, message) - - def handleEndHeaders(self): - self.factory.gotHeaders(self.headers) - - def connectionLost(self, reason): - self._connection_lost_reason = reason - HTTPClient.connectionLost(self, reason) - self.factory.noPage(reason) - - def handleResponse(self, response): - if self.factory.method.upper() == b"HEAD": - self.factory.page(b"") - elif self.length is not None and self.length > 0: - self.factory.noPage(self._connection_lost_reason) - else: - self.factory.page(response) - self.transport.loseConnection() - - def timeout(self): - self.transport.loseConnection() - - # transport cleanup needed for HTTPS connections - if self.factory.url.startswith(b"https"): - self.transport.stopProducing() - - self.factory.noPage( - DownloadTimeoutError( - f"Getting {self.factory.url} took longer " - f"than {self.factory.timeout} seconds." - ) - ) - - -# This class used to inherit from Twisted’s -# twisted.web.client.HTTPClientFactory. When that class was deprecated in -# Twisted (https://github.com/twisted/twisted/pull/643), we merged its -# non-overridden code into this class. -class ScrapyHTTPClientFactory(ClientFactory): - protocol = ScrapyHTTPPageGetter - - waiting = 1 - noisy = False - followRedirect = False - afterFoundGet = False - - def _build_response(self, body, request): - request.meta["download_latency"] = self.headers_time - self.start_time - status = int(self.status) - headers = Headers(self.response_headers) - respcls = responsetypes.from_args(headers=headers, url=self._url, body=body) - return respcls( - url=self._url, - status=status, - headers=headers, - body=body, - protocol=to_unicode(self.version), - ) - - def _set_connection_attributes(self, request): - proxy = request.meta.get("proxy") - if proxy: - proxy_parsed = urlparse(to_bytes(proxy, encoding="ascii")) - self.scheme = proxy_parsed.scheme - self.host = proxy_parsed.hostname - self.port = proxy_parsed.port - self.netloc = proxy_parsed.netloc - if self.port is None: - self.port = 443 if proxy_parsed.scheme == b"https" else 80 - self.path = self.url - else: - parsed = urlparse_cached(request) - path_str = urlunparse( - ("", "", parsed.path or "/", parsed.params, parsed.query, "") - ) - self.path = to_bytes(path_str, encoding="ascii") - assert parsed.hostname is not None - self.host = to_bytes(parsed.hostname, encoding="ascii") - self.port = parsed.port - self.scheme = to_bytes(parsed.scheme, encoding="ascii") - self.netloc = to_bytes(parsed.netloc, encoding="ascii") - if self.port is None: - self.port = 443 if self.scheme == b"https" else 80 - - def __init__(self, request: Request, timeout: float = 180): - warnings.warn( - "ScrapyHTTPClientFactory is deprecated and will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - self._url: str = urldefrag(request.url)[0] - # converting to bytes to comply to Twisted interface - self.url: bytes = to_bytes(self._url, encoding="ascii") - self.method: bytes = to_bytes(request.method, encoding="ascii") - self.body: bytes | None = request.body or None - self.headers: Headers = Headers(request.headers) - self.response_headers: Headers | None = None - self.timeout: float = request.meta.get("download_timeout") or timeout - self.start_time: float = time() - self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback( - self._build_response, request - ) - - # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected - # to have _disconnectedDeferred. See Twisted r32329. - # As Scrapy implements it's own logic to handle redirects is not - # needed to add the callback _waitForDisconnect. - # Specifically this avoids the AttributeError exception when - # clientConnectionFailed method is called. - self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred() - - self._set_connection_attributes(request) - - # set Host header based on url - self.headers.setdefault("Host", self.netloc) - - # set Content-Length based len of body - if self.body is not None: - self.headers["Content-Length"] = len(self.body) - # just in case a broken http/1.1 decides to keep connection alive - self.headers.setdefault("Connection", "close") - # Content-Length must be specified in POST method even with no body - elif self.method == b"POST": - self.headers["Content-Length"] = 0 - - def __repr__(self) -> str: - return f"<{self.__class__.__name__}: {self._url}>" - - def _cancelTimeout(self, result, timeoutCall): - if timeoutCall.active(): - timeoutCall.cancel() - return result - - def buildProtocol(self, addr): - p = ClientFactory.buildProtocol(self, addr) - p.followRedirect = self.followRedirect - p.afterFoundGet = self.afterFoundGet - if self.timeout: - from twisted.internet import reactor - - timeoutCall = reactor.callLater(self.timeout, p.timeout) - self.deferred.addBoth(self._cancelTimeout, timeoutCall) - return p - - def gotHeaders(self, headers): - self.headers_time = time() - self.response_headers = headers - - def gotStatus(self, version, status, message): - """ - Set the status of the request on us. - @param version: The HTTP version. - @type version: L{bytes} - @param status: The HTTP status code, an integer represented as a - bytestring. - @type status: L{bytes} - @param message: The HTTP status message. - @type message: L{bytes} - """ - self.version, self.status, self.message = version, status, message - - def page(self, page): - if self.waiting: - self.waiting = 0 - self.deferred.callback(page) - - def noPage(self, reason): - if self.waiting: - self.waiting = 0 - self.deferred.errback(reason) - - def clientConnectionFailed(self, _, reason): - """ - When a connection attempt fails, the request cannot be issued. If no - result has yet been provided to the result Deferred, provide the - connection failure reason as an error result. - """ - if self.waiting: - self.waiting = 0 - # If the connection attempt failed, there is nothing more to - # disconnect, so just fire that Deferred now. - self._disconnectedDeferred.callback(None) - self.deferred.errback(reason) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dc4106963..1033e874f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -8,8 +8,10 @@ For more information see docs/topics/architecture.rst from __future__ import annotations import asyncio +import contextlib import logging import warnings +from functools import partial from time import time from traceback import format_exc from typing import TYPE_CHECKING, Any @@ -197,7 +199,8 @@ class ExecutionEngine: self._start_request_processing_awaitable = asyncio.ensure_future(coro) else: self._start_request_processing_awaitable = Deferred.fromCoroutine(coro) - await maybe_deferred_to_future(self._closewait) + with contextlib.suppress(asyncio.exceptions.CancelledError): + await maybe_deferred_to_future(self._closewait) def stop(self) -> Deferred[None]: # pragma: no cover warnings.warn( @@ -271,7 +274,7 @@ class ExecutionEngine: """ assert self._start is not None try: - item_or_request = await self._start.__anext__() + item_or_request = await anext(self._start) except StopAsyncIteration: self._start = None except Exception as exception: @@ -350,6 +353,10 @@ class ExecutionEngine: or self.scraper.slot.needs_backout() ) + def _remove_request(self, _: Any, request: Request) -> None: + assert self._slot + self._slot.remove_request(request) + def _start_scheduled_request(self) -> bool: assert self._slot is not None # typing assert self.spider is not None # typing @@ -369,11 +376,7 @@ class ExecutionEngine: ) ) - def _remove_request(_: Any) -> None: - assert self._slot - self._slot.remove_request(request) - - d2: Deferred[None] = d.addBoth(_remove_request) + d2: Deferred[None] = d.addBoth(partial(self._remove_request, request=request)) d2.addErrback( lambda f: logger.info( "Error while removing request from slot", @@ -610,30 +613,33 @@ class ExecutionEngine: "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} ) - def log_failure(msg: str) -> None: - logger.error(msg, exc_info=True, extra={"spider": spider}) # noqa: LOG014 - try: await self._slot.close() except Exception: - log_failure("Slot close failure") + logger.error("Slot close failure", exc_info=True, extra={"spider": spider}) try: self.downloader.close() except Exception: - log_failure("Downloader close failure") + logger.error( + "Downloader close failure", exc_info=True, extra={"spider": spider} + ) try: await self.scraper.close_spider_async() except Exception: - log_failure("Scraper close failure") + logger.error( + "Scraper close failure", exc_info=True, extra={"spider": spider} + ) if hasattr(self._slot.scheduler, "close"): try: if (d := self._slot.scheduler.close(reason)) is not None: await maybe_deferred_to_future(d) except Exception: - log_failure("Scheduler close failure") + logger.error( + "Scheduler close failure", exc_info=True, extra={"spider": spider} + ) try: await self.signals.send_catch_log_async( @@ -642,7 +648,11 @@ class ExecutionEngine: reason=reason, ) except Exception: - log_failure("Error while sending spider_close signal") + logger.error( + "Error while sending spider_close signal", + exc_info=True, + extra={"spider": spider}, + ) assert self.crawler.stats try: @@ -659,7 +669,7 @@ class ExecutionEngine: else: self.crawler.stats.close_spider(reason=reason) except Exception: - log_failure("Stats close failure") + logger.error("Stats close failure") logger.info( "Spider closed (%(reason)s)", @@ -673,4 +683,4 @@ class ExecutionEngine: try: await ensure_awaitable(self._spider_closed_callback(spider)) except Exception: - log_failure("Error running spider_closed_callback") + logger.error("Error running spider_closed_callback") diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 8760b13ec..7137a0f2b 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -14,7 +14,7 @@ from twisted.web.client import ( ) from twisted.web.error import SchemeNotSupported -from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory +from scrapy.core.downloader.contextfactory import _AcceptableProtocolsContextFactory from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol if TYPE_CHECKING: @@ -43,6 +43,10 @@ class H2ConnectionPool: ConnectionKeyT, deque[Deferred[H2ClientProtocol]] ] = {} + self._tls_verbose_logging: bool = settings.getbool( + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" + ) + def get_connection( self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint ) -> Deferred[H2ClientProtocol]: @@ -71,7 +75,12 @@ class H2ConnectionPool: conn_lost_deferred: Deferred[list[BaseException]] = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) - factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) + factory = H2ClientFactory( + uri, + self.settings, + conn_lost_deferred, + tls_verbose_logging=self._tls_verbose_logging, + ) conn_d = endpoint.connect(factory) conn_d.addCallback(self.put_connection, key) @@ -122,11 +131,11 @@ class H2Agent: pool: H2ConnectionPool, context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008 connect_timeout: float | None = None, - bind_address: bytes | None = None, + bind_address: tuple[str, int] | None = None, ) -> None: self._reactor = reactor self._pool = pool - self._context_factory = AcceptableProtocolsContextFactory( + self._context_factory = _AcceptableProtocolsContextFactory( context_factory, acceptable_protocols=[b"h2"] ) self.endpoint_factory = _StandardEndpointFactory( @@ -134,7 +143,7 @@ class H2Agent: ) def get_endpoint(self, uri: URI) -> HostnameEndpoint: - return self.endpoint_factory.endpointForURI(uri) + return self.endpoint_factory.endpointForURI(uri) # type: ignore[no-any-return] def get_key(self, uri: URI) -> ConnectionKeyT: """ @@ -156,30 +165,3 @@ class H2Agent: lambda conn: conn.request(request, spider) ) return d2 - - -class ScrapyProxyH2Agent(H2Agent): - def __init__( - self, - reactor: ReactorBase, - proxy_uri: URI, - pool: H2ConnectionPool, - context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008 - connect_timeout: float | None = None, - bind_address: bytes | None = None, - ) -> None: - super().__init__( - reactor=reactor, - pool=pool, - context_factory=context_factory, - connect_timeout=connect_timeout, - bind_address=bind_address, - ) - self._proxy_uri = proxy_uri - - def get_endpoint(self, uri: URI) -> HostnameEndpoint: - return self.endpoint_factory.endpointForURI(self._proxy_uri) - - def get_key(self, uri: URI) -> ConnectionKeyT: - """We use the proxy uri instead of uri obtained from request url""" - return b"http-proxy", self._proxy_uri.host, self._proxy_uri.port diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index fe4128a68..39703f976 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -35,11 +35,11 @@ from scrapy.core.http2.stream import Stream, StreamCloseReason from scrapy.exceptions import DownloadTimeoutError from scrapy.http import Request, Response from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute +from scrapy.utils.ssl import _log_ssl_conn_debug_info if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address - from hpack import HeaderTuple from twisted.internet.defer import Deferred from twisted.python.failure import Failure from twisted.web.client import URI @@ -92,6 +92,8 @@ class H2ClientProtocol(Protocol, TimeoutMixin): uri: URI, settings: Settings, conn_lost_deferred: Deferred[list[BaseException]], + *, + tls_verbose_logging: bool = False, ) -> None: """ Arguments: @@ -101,8 +103,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): settings -- Scrapy project settings conn_lost_deferred -- Deferred fires with the reason: Failure to notify that connection was lost + tls_verbose_logging -- Whether to log TLS details """ self._conn_lost_deferred: Deferred[list[BaseException]] = conn_lost_deferred + self._tls_verbose_logging: bool = tls_verbose_logging config = H2Configuration(client_side=True, header_encoding="utf-8") self.conn = H2Connection(config=config) @@ -220,7 +224,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): """ assert self.transport is not None # typing # Reset the idle timeout as connection is still actively sending data - self.resetTimeout() + self.resetTimeout() # type: ignore[no-untyped-call] data = self.conn.data_to_send() self.transport.write(data) @@ -247,7 +251,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): sending some data now: we should open with the connection preamble. """ # Initialize the timeout - self.setTimeout(self.IDLE_TIMEOUT) + self.setTimeout(self.IDLE_TIMEOUT) # type: ignore[no-untyped-call] assert self.transport is not None # typing destination = self.transport.getPeer() @@ -260,7 +264,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def _lose_connection_with_error(self, errors: list[BaseException]) -> None: """Helper function to lose the connection with the error sent as a reason""" - self._conn_lost_errors += errors + self._conn_lost_errors.extend(errors) assert self.transport is not None # typing self.transport.loseConnection() @@ -278,6 +282,11 @@ class H2ClientProtocol(Protocol, TimeoutMixin): [InvalidNegotiatedProtocol(self.transport.negotiatedProtocol)] ) + if self._tls_verbose_logging: + connection = self.transport.getHandle() + hostname = self.metadata["uri"].host.decode("ascii") + _log_ssl_conn_debug_info(hostname, connection) + def _check_received_data(self, data: bytes) -> None: """Checks for edge cases where the connection to remote fails without raising an appropriate H2Error @@ -290,7 +299,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def dataReceived(self, data: bytes) -> None: # Reset the idle timeout as connection is still actively receiving data - self.resetTimeout() + self.resetTimeout() # type: ignore[no-untyped-call] try: self._check_received_data(data) @@ -300,7 +309,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if isinstance(e, FrameTooLargeError): # hyper-h2 does not drop the connection in this scenario, we # need to abort the connection manually. - self._conn_lost_errors += [e] + self._conn_lost_errors.append(e) assert self.transport is not None # typing self.transport.abortConnection() return @@ -343,7 +352,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): No need to write anything to transport here. """ # Cancel the timeout if not done yet - self.setTimeout(None) + self.setTimeout(None) # type: ignore[no-untyped-call] # Notify the connection pool instance such that no new requests are # sent over current connection @@ -409,7 +418,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): except KeyError: pass # We ignore server-initiated events else: - stream.receive_headers(cast("list[HeaderTuple]", event.headers)) + stream.receive_headers(cast("list[tuple[str, str]]", event.headers)) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: self.metadata["settings_acknowledged"] = True @@ -454,13 +463,21 @@ class H2ClientFactory(Factory): uri: URI, settings: Settings, conn_lost_deferred: Deferred[list[BaseException]], + *, + tls_verbose_logging: bool = False, ) -> None: self.uri = uri self.settings = settings self.conn_lost_deferred = conn_lost_deferred + self.tls_verbose_logging = tls_verbose_logging def buildProtocol(self, addr: IAddress) -> H2ClientProtocol: - return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred) + return H2ClientProtocol( + self.uri, + self.settings, + self.conn_lost_deferred, + tls_verbose_logging=self.tls_verbose_logging, + ) def acceptableProtocols(self) -> list[bytes]: return [PROTOCOL_NAME] diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 39c2eaae8..4d072c555 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -22,7 +22,7 @@ from scrapy.utils._download_handlers import ( from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: - from hpack import HeaderTuple + from collections.abc import Sequence from scrapy.core.http2.protocol import H2ClientProtocol from scrapy.http import Request, Response @@ -150,19 +150,21 @@ class Stream: # flow control window "flow_controlled_size": 0, # Headers received after sending the request - "headers": Headers({}), + "headers": Headers(), + # Response status code + "status": None, } - def _cancel(_: Any) -> None: - # Close this stream as gracefully as possible - # If the associated request is initiated we reset this stream - # else we directly call close() method - if self.metadata["request_sent"]: - self.reset_stream(StreamCloseReason.CANCELLED) - else: - self.close(StreamCloseReason.CANCELLED) + self._deferred_response: Deferred[Response] = Deferred(self._cancel) - self._deferred_response: Deferred[Response] = Deferred(_cancel) + def _cancel(self, _: Any) -> None: + # Close this stream as gracefully as possible + # If the associated request is initiated we reset this stream + # else we directly call close() method + if self.metadata["request_sent"]: + self.reset_stream(StreamCloseReason.CANCELLED) + else: + self.close(StreamCloseReason.CANCELLED) def __repr__(self) -> str: return f"Stream(id={self.stream_id!r})" @@ -359,9 +361,13 @@ class Stream: self._response["flow_controlled_size"], self.stream_id ) - def receive_headers(self, headers: list[HeaderTuple]) -> None: + def receive_headers(self, headers: list[tuple[str, str]]) -> None: for name, value in headers: - self._response["headers"].appendlist(name, value) + if name == ":status": + # it's a pseudo-header + self._response["status"] = int(value) + else: + self._response["headers"].appendlist(name, value) # Check if we exceed the allowed max data size which can be received expected_size = int(self._response["headers"].get(b"Content-Length", -1)) @@ -391,7 +397,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: list[BaseException] | None = None, + errors: Sequence[BaseException] | None = None, from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case.""" @@ -405,7 +411,7 @@ class Stream: # Have default value of errors as an empty list as # some cases can add a list of exceptions - errors = errors or [] + errors = errors or () if not from_protocol: self._protocol.pop_stream(self.stream_id) @@ -451,7 +457,8 @@ class Stream: # There maybe no :status in headers, we make # HTTP Status Code: 499 - Client Closed Request - self._response["headers"][":status"] = "499" + if self._response["status"] is None: + self._response["status"] = 499 self._fire_response_deferred() elif reason is StreamCloseReason.RESET: @@ -470,7 +477,7 @@ class Stream: self._deferred_response.errback(ResponseFailed(errors)) elif reason is StreamCloseReason.INACTIVE: - errors.insert(0, InactiveStreamClosed(self._request)) + errors = (InactiveStreamClosed(self._request), *errors) self._deferred_response.errback(ResponseFailed(errors)) else: @@ -490,7 +497,7 @@ class Stream: response = make_response( url=self._request.url, - status=int(self._response["headers"][":status"]), + status=self._response["status"], headers=self._response["headers"], body=self._response["body"].getvalue(), certificate=self._protocol.metadata["certificate"], diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 5a0aa2197..7217da942 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,17 +4,14 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any -from warnings import warn +from typing import TYPE_CHECKING, Any, cast # working around https://github.com/sphinx-doc/sphinx/issues/10400 from twisted.internet.defer import Deferred # noqa: TC002 -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.spiders import Spider # noqa: TC001 from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object -from scrapy.utils.python import global_object_name if TYPE_CHECKING: # requires queuelib >= 1.6.2 @@ -337,7 +334,7 @@ class Scheduler(BaseScheduler): cls = crawler.settings[f"SCHEDULER_START_{queue}_QUEUE"] if not cls: return None - return load_object(cls) + return cast("type[BaseQueue]", load_object(cls)) def has_pending_requests(self) -> bool: return len(self) > 0 @@ -450,27 +447,13 @@ class Scheduler(BaseScheduler): """Create a new priority queue instance, with in-memory storage""" assert self.crawler assert self.pqclass - try: - return build_from_crawler( - self.pqclass, - self.crawler, - downstream_queue_cls=self.mqclass, - key="", - start_queue_cls=self._smqclass, - ) - except TypeError: # pragma: no cover - warn( - f"The __init__ method of {global_object_name(self.pqclass)} " - f"does not support a `start_queue_cls` keyword-only " - f"parameter.", - ScrapyDeprecationWarning, - ) - return build_from_crawler( - self.pqclass, - self.crawler, - downstream_queue_cls=self.mqclass, - key="", - ) + return build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.mqclass, + key="", + start_queue_cls=self._smqclass, + ) def _dq(self) -> ScrapyPriorityQueue: """Create a new priority queue instance, with disk storage""" @@ -478,29 +461,14 @@ class Scheduler(BaseScheduler): assert self.dqdir assert self.pqclass state = self._read_dqs_state(self.dqdir) - try: - q = build_from_crawler( - self.pqclass, - self.crawler, - downstream_queue_cls=self.dqclass, - key=self.dqdir, - startprios=state, - start_queue_cls=self._sdqclass, - ) - except TypeError: # pragma: no cover - warn( - f"The __init__ method of {global_object_name(self.pqclass)} " - f"does not support a `start_queue_cls` keyword-only " - f"parameter.", - ScrapyDeprecationWarning, - ) - q = build_from_crawler( - self.pqclass, - self.crawler, - downstream_queue_cls=self.dqclass, - key=self.dqdir, - startprios=state, - ) + q = build_from_crawler( + self.pqclass, + self.crawler, + downstream_queue_cls=self.dqclass, + key=self.dqdir, + startprios=state, + start_queue_cls=self._sdqclass, + ) if q: logger.info( "Resuming crawl (%(queuesize)d requests scheduled)", @@ -521,7 +489,7 @@ class Scheduler(BaseScheduler): def _read_dqs_state(self, dqdir: str) -> Any: path = Path(dqdir, "active.json") if not path.exists(): - return [] + return () with path.open(encoding="utf-8") as f: return json.load(f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 695f6ef3e..466ce656d 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -23,7 +23,6 @@ from scrapy.exceptions import ( from scrapy.http import Request, Response from scrapy.pipelines import ItemPipelineManager from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available -from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.defer import ( _defer_sleep_async, _schedule_coro, @@ -66,7 +65,7 @@ class Slot: self.queue: deque[QueueTuple] = deque() self.active: set[Request] = set() self.active_size: int = 0 - self.itemproc_size: int = 0 + self.itemproc_size: int = 0 # just for scrapy.utils.engine.get_engine_status() self.closing: Deferred[Spider] | None = None def add_response_request( @@ -126,7 +125,7 @@ class Scraper: def _check_deprecated_itemproc_method(self, method: str) -> None: itemproc_cls = type(self.itemproc) - if not hasattr(self.itemproc, "process_item_async"): + if not hasattr(self.itemproc, f"{method}_async"): warnings.warn( f"{global_object_name(itemproc_cls)} doesn't define a {method}_async() method," f" this is deprecated and the method will be required in future Scrapy versions.", @@ -178,9 +177,7 @@ class Scraper: self.itemproc.open_spider(self.crawler.spider) ) - def close_spider( - self, spider: Spider | None = None - ) -> Deferred[None]: # pragma: no cover + def close_spider(self) -> Deferred[None]: # pragma: no cover warnings.warn( "Scraper.close_spider() is deprecated, use close_spider_async() instead", ScrapyDeprecationWarning, @@ -217,9 +214,8 @@ class Scraper: self.slot.closing.callback(self.crawler.spider) @inlineCallbacks - @_warn_spider_arg def enqueue_scrape( - self, result: Response | Failure, request: Request, spider: Spider | None = None + self, result: Response | Failure, request: Request ) -> Generator[Deferred[Any], Any, None]: if self.slot is None: raise RuntimeError("Scraper slot not assigned") @@ -349,13 +345,11 @@ class Scraper: ) return await ensure_awaitable(iterate_spider_output(output)) - @_warn_spider_arg def handle_spider_error( self, _failure: Failure, request: Request, response: Response | Failure, - spider: Spider | None = None, ) -> None: """Handle an exception raised by a spider callback or errback.""" assert self.crawler.spider @@ -391,7 +385,6 @@ class Scraper: result: Iterable[_T] | AsyncIterator[_T], request: Request, response: Response | Failure, - spider: Spider | None = None, ) -> Deferred[None]: # pragma: no cover """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" warnings.warn( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 007d03a04..fbc6f2530 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -9,31 +9,29 @@ from __future__ import annotations import logging from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from functools import wraps -from inspect import isasyncgenfunction, iscoroutine +from inspect import isasyncgenfunction from itertools import islice -from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar from warnings import warn -from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Request, Spider from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager -from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( _defer_sleep_async, deferred_from_coro, maybe_deferred_to_future, ) -from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_name +from scrapy.utils.python import MutableAsyncChain, global_object_name if TYPE_CHECKING: - from collections.abc import Generator + from twisted.internet.defer import Deferred - from scrapy.crawler import Crawler from scrapy.settings import BaseSettings @@ -47,95 +45,27 @@ ScrapeFunc: TypeAlias = Callable[ ] -def _isiterable(o: Any) -> bool: - return isinstance(o, (Iterable, AsyncIterator)) - - class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" @classmethod def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: - return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES")) - - def __init__(self, *middlewares: Any, crawler: Crawler | None = None) -> None: - self._check_deprecated_process_start_requests_use(middlewares) - super().__init__(*middlewares, crawler=crawler) - - def _check_deprecated_process_start_requests_use( - self, middlewares: tuple[Any, ...] - ) -> None: - deprecated_middlewares = [ - middleware - for middleware in middlewares - if hasattr(middleware, "process_start_requests") - and not hasattr(middleware, "process_start") - ] - modern_middlewares = [ - middleware - for middleware in middlewares - if not hasattr(middleware, "process_start_requests") - and hasattr(middleware, "process_start") - ] - if deprecated_middlewares and modern_middlewares: - raise ValueError( - "You are trying to combine spider middlewares that only " - "define the deprecated process_start_requests() method () " - "with spider middlewares that only define the " - "process_start() method (). This is not possible. You must " - "either disable or make universal 1 of those 2 sets of " - "spider middlewares. Making a spider middleware universal " - "means having it define both methods. See the release notes " - "of Scrapy 2.13 for details: " - "https://docs.scrapy.org/en/2.13/news.html" - ) - - self._use_start_requests = bool(deprecated_middlewares) - if self._use_start_requests: - deprecated_middleware_list = ", ".join( - global_object_name(middleware.__class__) - for middleware in deprecated_middlewares - ) - warn( - f"The following enabled spider middlewares, directly or " - f"through their parent classes, define the deprecated " - f"process_start_requests() method: " - f"{deprecated_middleware_list}. process_start_requests() has " - f"been deprecated in favor of a new method, process_start(), " - f"to support asynchronous code execution. " - f"process_start_requests() will stop being called in a future " - f"version of Scrapy. If you use Scrapy 2.13 or higher " - f"only, replace process_start_requests() with " - f"process_start(); note that process_start() is a coroutine " - f"(async def). If you need to maintain compatibility with " - f"lower Scrapy versions, when defining " - f"process_start_requests() in a spider middleware class, " - f"define process_start() as well. See the release notes of " - f"Scrapy 2.13 for details: " - f"https://docs.scrapy.org/en/2.13/news.html", - ScrapyDeprecationWarning, - ) + return build_component_list( + settings.get_component_priority_dict_with_base("SPIDER_MIDDLEWARES") + ) def _add_middleware(self, mw: Any) -> None: if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) self._check_mw_method_spider_arg(mw.process_spider_input) - if self._use_start_requests: - if hasattr(mw, "process_start_requests"): - self.methods["process_start_requests"].appendleft( - mw.process_start_requests - ) - elif hasattr(mw, "process_start"): + if hasattr(mw, "process_start"): self.methods["process_start"].appendleft(mw.process_start) - process_spider_output = self._get_async_method_pair(mw, "process_spider_output") + process_spider_output = self._get_process_spider_output(mw) self.methods["process_spider_output"].appendleft(process_spider_output) - if callable(process_spider_output): + if process_spider_output is not None: self._check_mw_method_spider_arg(process_spider_output) - elif isinstance(process_spider_output, tuple): - for m in process_spider_output: - self._check_mw_method_spider_arg(m) process_spider_exception = getattr(mw, "process_spider_exception", None) self.methods["process_spider_exception"].appendleft(process_spider_exception) @@ -149,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager): request: Request, ) -> Iterable[_T] | AsyncIterator[_T]: for method in self.methods["process_spider_input"]: - method = cast("Callable", method) + assert method is not None try: if method in self._mw_methods_requiring_spider: result = method(response=response, spider=self._spider) @@ -167,54 +97,28 @@ class SpiderMiddlewareManager(MiddlewareManager): return await scrape_func(Failure(), request) return await scrape_func(response, request) - def _evaluate_iterable( + async def _evaluate_iterable( self, response: Response, - iterable: Iterable[_T] | AsyncIterator[_T], + iterable: AsyncIterator[_T], exception_processor_index: int, - recover_to: MutableChain[_T] | MutableAsyncChain[_T], - ) -> Iterable[_T] | AsyncIterator[_T]: - def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: - try: - yield from iterable - except Exception as ex: - exception_result = cast( - "Failure | MutableChain[_T]", - self._process_spider_exception( - response, ex, exception_processor_index - ), - ) - if isinstance(exception_result, Failure): - raise - assert isinstance(recover_to, MutableChain) - recover_to.extend(exception_result) - - async def process_async(iterable: AsyncIterator[_T]) -> AsyncIterator[_T]: - try: - async for r in iterable: - yield r - except Exception as ex: - exception_result = cast( - "Failure | MutableAsyncChain[_T]", - self._process_spider_exception( - response, ex, exception_processor_index - ), - ) - if isinstance(exception_result, Failure): - raise - assert isinstance(recover_to, MutableAsyncChain) - recover_to.extend(exception_result) - - if isinstance(iterable, AsyncIterator): - return process_async(iterable) - return process_sync(iterable) + recover_to: MutableAsyncChain[_T], + ) -> AsyncIterator[_T]: + try: + async for r in iterable: + yield r + except Exception as ex: + exception_result: MutableAsyncChain[_T] = self._process_spider_exception( + response, ex, exception_processor_index + ) + recover_to.extend(exception_result) def _process_spider_exception( self, response: Response, exception: Exception, start_index: int = 0, - ) -> MutableChain[_T] | MutableAsyncChain[_T]: + ) -> MutableAsyncChain[_T]: # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): raise exception @@ -224,28 +128,18 @@ class SpiderMiddlewareManager(MiddlewareManager): for method_index, method in enumerate(method_list, start=start_index): if method is None: continue - method = cast("Callable", method) if method in self._mw_methods_requiring_spider: result = method( response=response, exception=exception, spider=self._spider ) else: result = method(response=response, exception=exception) - if _isiterable(result): + if isinstance(result, (Iterable, AsyncIterator)): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - dfd: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = ( - self._process_spider_output(response, result, method_index + 1) - ) - # _process_spider_output() returns a Deferred only because of downgrading so this can be - # simplified when downgrading is removed. - if dfd.called: - # the result is available immediately if _process_spider_output didn't do downgrading - return cast("MutableChain[_T] | MutableAsyncChain[_T]", dfd.result) - # we forbid waiting here because otherwise we would need to return a deferred from - # _process_spider_exception too, which complicates the architecture - msg = f"Async iterable returned from {global_object_name(method)} cannot be downgraded" - raise _InvalidOutput(msg) + if isinstance(result, Iterable): + result = as_async_generator(result) + return self._process_spider_output(response, result, method_index + 1) if result is None: continue msg = ( @@ -255,124 +149,35 @@ class SpiderMiddlewareManager(MiddlewareManager): raise _InvalidOutput(msg) raise exception - # This method cannot be made async def, as _process_spider_exception relies on the Deferred result - # being available immediately which doesn't work when it's a wrapped coroutine. - # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed. - @inlineCallbacks - def _process_spider_output( # noqa: PLR0912 + def _process_spider_output( self, response: Response, - result: Iterable[_T] | AsyncIterator[_T], + result: AsyncIterator[_T], start_index: int = 0, - ) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]: + ) -> MutableAsyncChain[_T]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - recovered: MutableChain[_T] | MutableAsyncChain[_T] - last_result_is_async = isinstance(result, AsyncIterator) - recovered = MutableAsyncChain() if last_result_is_async else MutableChain() - - # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. - # 1. def foo. Sync iterables are passed as is, async ones are downgraded. - # 2. async def foo. Sync iterables are upgraded, async ones are passed as is. - # 3. def foo + async def foo_async. Iterables are passed to the respective method. - # Storing methods and method tuples in the same list is weird but we should be able to roll this back - # when we drop this compatibility feature. - + recovered: MutableAsyncChain[_T] = MutableAsyncChain() method_list = islice(self.methods["process_spider_output"], start_index, None) - for method_index, method_pair in enumerate(method_list, start=start_index): - if method_pair is None: + for method_index, method in enumerate(method_list, start=start_index): + if method is None: continue - need_upgrade = need_downgrade = False - if isinstance(method_pair, tuple): - # This tuple handling is only needed until _async compatibility methods are removed. - method_sync, method_async = method_pair - method = method_async if last_result_is_async else method_sync + if method in self._mw_methods_requiring_spider: + result = method(response=response, result=result, spider=self._spider) else: - method = method_pair - if not last_result_is_async and isasyncgenfunction(method): - need_upgrade = True - elif last_result_is_async and not isasyncgenfunction(method): - need_downgrade = True - try: - if need_upgrade: - # Iterable -> AsyncIterator - result = as_async_generator(result) - elif need_downgrade: - logger.warning( - f"Async iterable passed to {global_object_name(method)} was" - f" downgraded to a non-async one. This is deprecated and will" - f" stop working in a future version of Scrapy. Please see" - f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" - f" for more information." - ) - assert isinstance(result, AsyncIterator) - # AsyncIterator -> Iterable - result = yield deferred_from_coro(collect_asyncgen(result)) - if isinstance(recovered, AsyncIterator): - recovered_collected = yield deferred_from_coro( - collect_asyncgen(recovered) - ) - recovered = MutableChain(recovered_collected) - # might fail directly if the output value is not a generator - if method in self._mw_methods_requiring_spider: - result = method( - response=response, result=result, spider=self._spider - ) - else: - result = method(response=response, result=result) - except Exception as ex: - exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = ( - self._process_spider_exception(response, ex, method_index + 1) - ) - if isinstance(exception_result, Failure): - raise - return exception_result - if _isiterable(result): - result = self._evaluate_iterable( - response, result, method_index + 1, recovered - ) - else: - if iscoroutine(result): - result.close() # Silence warning about not awaiting - msg = ( - f"{global_object_name(method)} must be an asynchronous " - f"generator (i.e. use yield)" - ) - else: - msg = ( - f"{global_object_name(method)} must return an iterable, got " - f"{type(result)}" - ) - raise _InvalidOutput(msg) - last_result_is_async = isinstance(result, AsyncIterator) - - if last_result_is_async: - return MutableAsyncChain(result, recovered) - return MutableChain(result, recovered) # type: ignore[arg-type] + result = method(response=response, result=result) + result = self._evaluate_iterable( + response, result, method_index + 1, recovered + ) + return MutableAsyncChain(result, recovered) async def _process_callback_output( - self, - response: Response, - result: Iterable[_T] | AsyncIterator[_T], - ) -> MutableChain[_T] | MutableAsyncChain[_T]: - recovered: MutableChain[_T] | MutableAsyncChain[_T] - if isinstance(result, AsyncIterator): - recovered = MutableAsyncChain() - else: - recovered = MutableChain() + self, response: Response, result: AsyncIterator[_T] + ) -> MutableAsyncChain[_T]: + recovered: MutableAsyncChain[_T] = MutableAsyncChain() result = self._evaluate_iterable(response, result, 0, recovered) - result = await maybe_deferred_to_future( - cast( - "Deferred[Iterable[_T] | AsyncIterator[_T]]", - self._process_spider_output(response, result), - ) - ) - if isinstance(result, AsyncIterator): - return MutableAsyncChain(result, recovered) - if isinstance(recovered, AsyncIterator): - recovered_collected = await collect_asyncgen(recovered) - recovered = MutableChain(recovered_collected) - return MutableChain(result, recovered) + result = self._process_spider_output(response, result) + return MutableAsyncChain(result, recovered) def scrape_response( self, @@ -383,7 +188,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: # pragma: no cover + ) -> Deferred[MutableAsyncChain[_T]]: # pragma: no cover warn( "SpiderMiddlewareManager.scrape_response() is deprecated, use scrape_response_async() instead", ScrapyDeprecationWarning, @@ -406,31 +211,21 @@ class SpiderMiddlewareManager(MiddlewareManager): scrape_func: ScrapeFunc[_T], response: Response, request: Request, - ) -> MutableChain[_T] | MutableAsyncChain[_T]: + ) -> MutableAsyncChain[_T]: if not self.crawler: raise RuntimeError( "scrape_response_async() called on a SpiderMiddlewareManager" " instance created without a crawler." ) - - async def process_callback_output( - result: Iterable[_T] | AsyncIterator[_T], - ) -> MutableChain[_T] | MutableAsyncChain[_T]: - return await self._process_callback_output(response, result) - - def process_spider_exception( - exception: Exception, - ) -> MutableChain[_T] | MutableAsyncChain[_T]: - return self._process_spider_exception(response, exception) - try: it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input( scrape_func, response, request ) - return await process_callback_output(it) + ait = it if isinstance(it, AsyncIterator) else as_async_generator(it) + return await self._process_callback_output(response, ait) except Exception as ex: await _defer_sleep_async() - return process_spider_exception(ex) + return self._process_spider_exception(response, ex) async def process_start( self, spider: Spider | None = None @@ -448,111 +243,32 @@ class SpiderMiddlewareManager(MiddlewareManager): ) warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) self._set_compat_spider(spider) - self._check_deprecated_start_requests_use() - if self._use_start_requests: - sync_start = iter(self._spider.start_requests()) - sync_start = await self._process_chain( - "process_start_requests", sync_start, always_add_spider=True - ) - start: AsyncIterator[Any] = as_async_generator(sync_start) - else: - start = self._spider.start() - start = await self._process_chain("process_start", start) - return start - - def _check_deprecated_start_requests_use(self) -> None: - start_requests_cls = None - start_cls = None - spidercls = self._spider.__class__ - mro = spidercls.__mro__ - - for cls in mro: - cls_dict = cls.__dict__ - if start_requests_cls is None and "start_requests" in cls_dict: - start_requests_cls = cls - if start_cls is None and "start" in cls_dict: - start_cls = cls - if start_requests_cls is not None and start_cls is not None: - break - - # Spider defines both, start_requests and start. - assert start_requests_cls is not None - assert start_cls is not None - - if ( - start_requests_cls is not Spider - and start_cls is not start_requests_cls - and mro.index(start_requests_cls) < mro.index(start_cls) - ): - src = global_object_name(start_requests_cls) - if start_requests_cls is not spidercls: - src += f" (inherited by {global_object_name(spidercls)})" - warn( - f"{src} defines the deprecated start_requests() method. " - f"start_requests() has been deprecated in favor of a new " - f"method, start(), to support asynchronous code " - f"execution. start_requests() will stop being called in a " - f"future version of Scrapy. If you use Scrapy 2.13 or " - f"higher only, replace start_requests() with start(); " - f"note that start() is a coroutine (async def). If you " - f"need to maintain compatibility with lower Scrapy versions, " - f"when overriding start_requests() in a spider class, " - f"override start() as well; you can use super() to " - f"reuse the inherited start() implementation without " - f"copy-pasting. See the release notes of Scrapy 2.13 for " - f"details: https://docs.scrapy.org/en/2.13/news.html", - ScrapyDeprecationWarning, - ) - - if ( - self._use_start_requests - and start_cls is not Spider - and start_requests_cls is not start_cls - and mro.index(start_cls) < mro.index(start_requests_cls) - ): - src = global_object_name(start_cls) - if start_cls is not spidercls: - src += f" (inherited by {global_object_name(spidercls)})" - raise ValueError( - f"{src} does not define the deprecated start_requests() " - f"method. However, one or more of your enabled spider " - f"middlewares (reported in an earlier deprecation warning) " - f"define the process_start_requests() method, and not the " - f"process_start() method, making them only compatible with " - f"(deprecated) spiders that define the start_requests() " - f"method. To solve this issue, disable the offending spider " - f"middlewares, upgrade them as described in that earlier " - f"deprecation warning, or make your spider compatible with " - f"deprecated spider middlewares (and earlier Scrapy versions) " - f"by defining a sync start_requests() method that works " - f"similarly to its existing start() method. See the " - f"release notes of Scrapy 2.13 for details: " - f"https://docs.scrapy.org/en/2.13/news.html" - ) + start = self._spider.start() + return await self._process_chain("process_start", start) # This method is only needed until _async compatibility methods are removed. @staticmethod - def _get_async_method_pair( - mw: Any, methodname: str - ) -> Callable | tuple[Callable, Callable] | None: - normal_method: Callable | None = getattr(mw, methodname, None) - methodname_async = methodname + "_async" - async_method: Callable | None = getattr(mw, methodname_async, None) + def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None: + normal_method: Callable[..., Any] | None = getattr( + mw, "process_spider_output", None + ) + async_method: Callable[..., Any] | None = getattr( + mw, "process_spider_output_async", None + ) if not async_method: if normal_method and not isasyncgenfunction(normal_method): - logger.warning( + raise TypeError( f"Middleware {global_object_name(mw.__class__)} doesn't support" - f" asynchronous spider output, this is deprecated and will stop" - f" working in a future version of Scrapy. The middleware should" - f" be updated to support it. Please see" - f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" - f" for more information." + f" asynchronous spider output. Its process_spider_output() method" + f" should be an async generator function or it should additionally" + f" define a process_spider_output_async() method." ) return normal_method if not normal_method: logger.error( - f"Middleware {global_object_name(mw.__class__)} has {methodname_async} " - f"without {methodname}, skipping this method." + f"Middleware {global_object_name(mw.__class__)} has" + f" process_spider_output_async() without process_spider_output()," + f" skipping this method. Please rename it to process_spider_output()." ) return None if not isasyncgenfunction(async_method): @@ -564,8 +280,8 @@ class SpiderMiddlewareManager(MiddlewareManager): if isasyncgenfunction(normal_method): logger.error( f"{global_object_name(normal_method)} is an async " - f"generator function while {methodname_async} exists, " - f"skipping both methods." + f"generator function while process_spider_output_async() exists, " + f"skipping both methods. Please remove process_spider_output_async()." ) return None - return normal_method, async_method + return async_method diff --git a/scrapy/crawler.py b/scrapy/crawler.py index b2f08450b..9828fe6f5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -7,6 +7,7 @@ import pprint import signal import warnings from abc import ABC, abstractmethod +from functools import partial from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks @@ -16,7 +17,7 @@ from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager -from scrapy.settings import Settings, overridden_settings +from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader from scrapy.utils.defer import deferred_from_coro @@ -105,18 +106,28 @@ class Crawler: self, ) - use_reactor = self.settings.getbool("TWISTED_ENABLED") + use_reactor = self.settings.getbool("TWISTED_REACTOR_ENABLED") if use_reactor: + # We either install a reactor or expect one to be installed. reactor_class: str = self.settings["TWISTED_REACTOR"] event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] if self._init_reactor: - # this needs to be done after the spider settings are merged, - # but before something imports twisted.internet.reactor + # We need to install a reactor. + # This needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor. if reactor_class: + # Install a specific reactor. install_reactor(reactor_class, event_loop) else: + # Install the default one. from twisted.internet import reactor # noqa: F401 + elif not is_reactor_installed(): + # We need a reactor to be already installed. + raise RuntimeError( + "We expected a Twisted reactor to be installed but it isn't." + ) if reactor_class: + # We need to check that the correct reactor is installed. verify_installed_reactor(reactor_class) if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) @@ -124,6 +135,11 @@ class Crawler: if self._init_reactor or reactor_class: log_reactor_info() else: + # We expect a reactor to not be installed. + if is_reactor_installed(): + raise RuntimeError( + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." + ) logger.debug("Not using a Twisted reactor") self._apply_reactorless_default_settings() @@ -143,6 +159,11 @@ class Crawler: change them here when the reactor is not used. """ self.settings.set("TELNETCONSOLE_ENABLED", False, priority="default") + for scheme in ("http", "https"): + self.settings["DOWNLOAD_HANDLERS_BASE"][scheme] = ( + "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler" + ) + self.settings["DOWNLOAD_HANDLERS_BASE"]["ftp"] = None # Cannot use @deferred_f_from_coro_f because that relies on the reactor # being installed already, which is done within _apply_settings(), inside @@ -342,9 +363,12 @@ class CrawlerRunnerBase(ABC): """ Return a :class:`~scrapy.crawler.Crawler` object. - * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. + * If ``crawler_or_spidercls`` is a Crawler, the runner's settings are + merged into it as defaults: for each setting, the runner's value + is applied only if the Crawler does not already have that setting at + an equal or higher priority. The Crawler is then returned. * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler - is constructed for it. + is constructed for it using this runner's settings. * If ``crawler_or_spidercls`` is a string, this function finds a spider with this name in a Scrapy project (using spider loader), then creates a Crawler instance for it. @@ -355,6 +379,7 @@ class CrawlerRunnerBase(ABC): "it must be a spider class (or a Crawler object)" ) if isinstance(crawler_or_spidercls, Crawler): + crawler_or_spidercls.settings.update(self.settings) return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) @@ -391,9 +416,9 @@ class CrawlerRunner(CrawlerRunnerBase): def __init__(self, settings: dict[str, Any] | Settings | None = None): super().__init__(settings) - if not self.settings.getbool("TWISTED_ENABLED"): + if not self.settings.getbool("TWISTED_REACTOR_ENABLED"): raise RuntimeError( - f"{type(self).__name__} doesn't support TWISTED_ENABLED=False." + f"{type(self).__name__} doesn't support TWISTED_REACTOR_ENABLED=False." ) self._active: set[Deferred[None]] = set() @@ -473,17 +498,24 @@ class CrawlerRunner(CrawlerRunnerBase): class AsyncCrawlerRunner(CrawlerRunnerBase): """ This is a convenient helper class that keeps track of, manages and runs - crawlers inside an already setup :mod:`~twisted.internet.reactor`. + crawlers inside an already setup :mod:`~twisted.internet.reactor` or + asyncio event loop. The AsyncCrawlerRunner object must be instantiated with a :class:`~scrapy.settings.Settings` object. + When the :setting:`TWISTED_REACTOR_ENABLED` setting is set to ``True``, + this class requires a reactor to be installed and uses it, otherwise it + requires a reactor to not be installed but requires an asyncio event loop + to be installed and uses it. + This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. This class provides coroutine APIs. It requires - :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` when used + with a reactor. """ def __init__(self, settings: dict[str, Any] | Settings | None = None): @@ -523,42 +555,48 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): "The crawler_or_spidercls argument cannot be a spider object, " "it must be a spider class (or a Crawler object)" ) - if self.settings.getbool("TWISTED_ENABLED"): + if self.settings.getbool("TWISTED_REACTOR_ENABLED"): + if not is_reactor_installed(): + raise RuntimeError( + "We expected a Twisted reactor to be installed but it isn't." + ) if not is_asyncio_reactor_installed(): raise RuntimeError( - f"When TWISTED_ENABLED is True, {type(self).__name__} " + f"When TWISTED_REACTOR_ENABLED is True, {type(self).__name__} " f"requires that the installed Twisted reactor is " f'"twisted.internet.asyncioreactor.AsyncioSelectorReactor".' ) elif is_reactor_installed(): raise RuntimeError( - "TWISTED_ENABLED is False but a Twisted reactor is installed." + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." ) crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) + async def _crawl_and_track( + self, crawler: Crawler, *args: Any, **kwargs: Any + ) -> None: + try: + await crawler.crawl_async(*args, **kwargs) + except Exception: + self.bootstrap_failed = True + raise # re-raise so asyncio still logs it to stderr naturally + + def _done(self, task: asyncio.Task[None], crawler: Crawler) -> None: + self._active.discard(task) + self.crawlers.discard(crawler) + self.bootstrap_failed |= not getattr(crawler, "spider", None) + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]: # At this point the asyncio loop has been installed either by the user # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()). loop = asyncio.get_event_loop() self.crawlers.add(crawler) - async def _crawl_and_track() -> None: - try: - await crawler.crawl_async(*args, **kwargs) - except Exception: - self.bootstrap_failed = True - raise # re-raise so asyncio still logs it to stderr naturally - - task = loop.create_task(_crawl_and_track()) + task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs)) self._active.add(task) + task.add_done_callback(partial(self._done, crawler=crawler)) - def _done(_: asyncio.Task[None]) -> None: - self.crawlers.discard(crawler) - self._active.discard(task) - self.bootstrap_failed |= not getattr(crawler, "spider", None) - - task.add_done_callback(_done) return task async def stop(self) -> None: @@ -601,30 +639,60 @@ class CrawlerProcessBase(CrawlerRunnerBase): from twisted.internet import reactor install_shutdown_handlers(self._signal_kill) - signame = signal_names[signum] - logger.info( - "Received %(signame)s, shutting down gracefully. Send again to force ", - {"signame": signame}, - ) + self._log_shutdown(signum) reactor.callFromThread(self._graceful_stop_reactor) def _signal_kill(self, signum: int, _: Any) -> None: from twisted.internet import reactor install_shutdown_handlers(signal.SIG_IGN) + self._log_kill(signum) + reactor.callFromThread(self._stop_reactor) + + @staticmethod + def _log_shutdown(signum: int) -> None: + signame = signal_names[signum] + logger.info( + "Received %(signame)s, shutting down gracefully. Send again to force ", + {"signame": signame}, + ) + + @staticmethod + def _log_kill(signum: int) -> None: signame = signal_names[signum] logger.info( "Received %(signame)s twice, forcing unclean shutdown", {"signame": signame} ) - reactor.callFromThread(self._stop_reactor) def _setup_reactor(self, install_signal_handlers: bool) -> None: from twisted.internet import reactor - resolver_class = load_object(self.settings["DNS_RESOLVER"]) + dns_priority = self.settings.getpriority("DNS_RESOLVER") or 0 + default_priority = SETTINGS_PRIORITIES["default"] + + if dns_priority > default_priority: + warnings.warn( + "The DNS_RESOLVER setting is deprecated, please use " + "TWISTED_DNS_RESOLVER instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + twisted_dns_priority = ( + self.settings.getpriority("TWISTED_DNS_RESOLVER") or 0 + ) + if twisted_dns_priority > dns_priority: + resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"] + else: + resolver_cls_path = self.settings["DNS_RESOLVER"] + else: + resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"] + + resolver_class = load_object(resolver_cls_path) + # We pass self, which is CrawlerProcess, instead of Crawler here, # which works because the default resolvers only use crawler.settings. - resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] + resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[call-overload] resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) @@ -704,8 +772,8 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner): ) -> None: """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool - size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache - based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. + size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS + resolver based on :setting:`DNSCACHE_ENABLED`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. @@ -745,6 +813,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): The AsyncCrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. + When the :setting:`TWISTED_REACTOR_ENABLED` setting is set to ``True``, + this class installs a reactor and uses it, otherwise it requires a reactor + to not be installed but installs an asyncio event loop and uses it. + :param install_root_handler: whether to install root logging handler (default: True) @@ -753,7 +825,8 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): process. See :ref:`run-from-script` for an example. This class provides coroutine APIs. It requires - :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` when used + with a reactor. """ def __init__( @@ -763,18 +836,19 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): ): super().__init__(settings, install_root_handler) logger.debug("Using AsyncCrawlerProcess") + self._reactorless_loop: asyncio.AbstractEventLoop | None = None # We want the asyncio event loop to be installed early, so that it's # always the correct one. And as we do that, we can also install the # reactor here. # The ASYNCIO_EVENT_LOOP setting cannot be overridden by add-ons and # spiders when using AsyncCrawlerProcess. loop_path = self.settings["ASYNCIO_EVENT_LOOP"] - if not self.settings.getbool("TWISTED_ENABLED"): + if not self.settings.getbool("TWISTED_REACTOR_ENABLED"): if is_reactor_installed(): raise RuntimeError( - "TWISTED_ENABLED is False but a Twisted reactor is installed." + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." ) - set_asyncio_event_loop(loop_path) + self._reactorless_loop = set_asyncio_event_loop(loop_path) install_reactor_import_hook() elif is_reactor_installed(): # The user could install a reactor before this class is instantiated. @@ -786,6 +860,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): else: install_reactor(_asyncio_reactor_path, loop_path) self._initialized_reactor = True + self._reactorless_main_task: asyncio.Future[None] | None = None def _stop_dfd(self) -> Deferred[Any]: return deferred_from_coro(self.stop()) @@ -794,9 +869,13 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): self, stop_after_crawl: bool = True, install_signal_handlers: bool = True ) -> None: """ - This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool - size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache - based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. + This method starts a :mod:`~twisted.internet.reactor` or an asyncio + event loop, depending on the value of the + :setting:`TWISTED_REACTOR_ENABLED` setting. + + When using a reactor it adjusts its pool size to + :setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based + on :setting:`DNSCACHE_ENABLED`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. @@ -808,7 +887,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): handlers from Twisted and Scrapy (default: True) """ - if not self.settings.getbool("TWISTED_ENABLED"): + if not self.settings.getbool("TWISTED_REACTOR_ENABLED"): self._start_asyncio(stop_after_crawl, install_signal_handlers) else: self._start_twisted(stop_after_crawl, install_signal_handlers) @@ -816,20 +895,150 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): def _start_asyncio( self, stop_after_crawl: bool, install_signal_handlers: bool ) -> None: - # Very basic and will need multiple improvements. - # TODO https://docs.python.org/3/library/asyncio-runner.html#handling-keyboard-interruption - # TODO various exception handling - # TODO consider asyncio.run() + # We cannot use asyncio.run() here, because we can't let it handle the + # loop lifetime: _crawl() needs a loop (which we create in __init__()), + # because crawl() returns a Task. + # So we reproduce a part of asyncio.runners.Runner that is useful to us. + + # Normal workflow: + # 1. _start_asyncio() creates a task for self.join() and calls _run_loop() + # 2. _run_loop() calls loop.run_until_complete(main_task) + # 3. Crawling tasks start and finish + # 4. join() completes, loop.run_until_complete() and thus _run_loop() return + # 5. _start_asyncio() calls _close_loop() + # 6. _close_loop() does finalization and calls loop.close() + + # Normal workflow with stop_after_crawl=False: + # 1. _start_asyncio() creates a simple future and calls _run_loop() + # 2. _run_loop() calls loop.run_until_complete(main_task) + # 3. Crawling tasks start and finish + # 4. _run_loop() blocks until the loop is stopped externally or the + # future is cancelled via Ctrl-C + # 5. (after _run_loop() returns) _start_asyncio() calls _close_loop() + # 6. _close_loop() does finalization and calls loop.close() + + # Workflow with Ctrl-C pressed once: + # 1. While loop.run_until_complete() blocks, _signal_shutdown_reactorless() + # is called + # 2. _signal_shutdown_reactorless() calls _shutdown_graceful_reactorless() + # (via call_soon_threadsafe()) + # 3. _shutdown_graceful_reactorless() calls stop() + # 4. For stop_after_crawl=True: crawl tasks finish, join() completes, + # loop.run_until_complete() and thus _run_loop() return + # For stop_after_crawl=False: _shutdown_graceful_reactorless() waits + # for crawl tasks via join(), then cancels the main task, + # loop.run_until_complete() raises CancelledError, _run_loop() returns + # 5. _start_asyncio() calls _close_loop() + # 6. _close_loop() does finalization and calls loop.close() + + # Workflow with Ctrl-C pressed twice: + # 1. While loop.run_until_complete() blocks, _signal_shutdown_reactorless() + # is called + # 2. _signal_shutdown_reactorless() calls _shutdown_graceful_reactorless() + # (via call_soon_threadsafe()) and installs _signal_kill_reactorless() + # as the next handler + # 3. Before _shutdown_graceful_reactorless() completes, + # _signal_kill_reactorless() is called + # 4. _signal_kill_reactorless() cancels the main task + # (via call_soon_threadsafe()) + # 5. loop.run_until_complete() raises CancelledError, _run_loop() returns + # 6. _start_asyncio() calls _close_loop() + # 7. _close_loop() cancels all pending tasks (including + # _shutdown_graceful_reactorless()), does finalization and calls loop.close() + + loop = self._reactorless_loop + assert loop - loop = asyncio.get_event_loop() if stop_after_crawl: - join_task = loop.create_task(self.join()) - join_task.add_done_callback(lambda _: loop.stop()) + self._reactorless_main_task = loop.create_task(self.join()) + else: + self._reactorless_main_task = loop.create_future() + self._stop_after_crawl = stop_after_crawl + try: - loop.run_forever() # blocking call + self._run_loop(install_signal_handlers) # blocking call + except asyncio.CancelledError: + pass finally: + self._close_loop() + + def _run_loop(self, install_signal_handlers: bool) -> None: + # similar to asyncio.runners.Runner.run() + if install_signal_handlers: + install_shutdown_handlers(self._signal_shutdown_reactorless) + assert self._reactorless_loop + assert self._reactorless_main_task + self._reactorless_loop.run_until_complete(self._reactorless_main_task) + + def _close_loop(self) -> None: + # Similar to asyncio.runners.Runner.close() + loop = self._reactorless_loop + assert loop + try: + self._cancel_all_tasks(loop) loop.run_until_complete(loop.shutdown_asyncgens()) + loop.run_until_complete(loop.shutdown_default_executor()) + finally: + self._reactorless_main_task = None + asyncio.set_event_loop(None) loop.close() + self._reactorless_loop = None + + @staticmethod + def _cancel_all_tasks(loop: asyncio.AbstractEventLoop) -> None: + # copy of asyncio.runners._cancel_all_tasks() + to_cancel = asyncio.all_tasks(loop) + if not to_cancel: + return + + for task in to_cancel: + task.cancel() + + loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True)) + + for task in to_cancel: + if task.cancelled(): + continue + if task.exception() is not None: + loop.call_exception_handler( + { + "message": "unhandled exception during AsyncCrawlerProcess shutdown", + "exception": task.exception(), + "task": task, + } + ) + + def _signal_shutdown_reactorless(self, signum: int, _: Any) -> None: + install_shutdown_handlers(self._signal_kill_reactorless) + self._log_shutdown(signum) + if (loop := self._reactorless_loop) is None: + return + + loop.call_soon_threadsafe(self._create_shutdown_task) + + def _create_shutdown_task(self) -> None: + assert self._reactorless_loop + coro = self._shutdown_graceful_reactorless() + try: + self._reactorless_loop.create_task(coro) + except RuntimeError: + coro.close() + + async def _shutdown_graceful_reactorless(self) -> None: + await self.stop() + if not self._stop_after_crawl: + # wait until crawl tasks finish and cancel the future + await self.join() + if self._reactorless_main_task and not self._reactorless_main_task.done(): + self._reactorless_main_task.cancel() + + def _signal_kill_reactorless(self, signum: int, _: Any) -> None: + install_shutdown_handlers(signal.SIG_IGN) + self._log_kill(signum) + if (loop := self._reactorless_loop) is None: + return + if (task := self._reactorless_main_task) is not None: + loop.call_soon_threadsafe(task.cancel) def _start_twisted( self, stop_after_crawl: bool, install_signal_handlers: bool diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py deleted file mode 100644 index a23deaa45..000000000 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import annotations - -import logging -import re -from typing import TYPE_CHECKING -from warnings import warn - -from w3lib import html - -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import HtmlResponse, Response -from scrapy.utils.url import escape_ajax - -if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from typing_extensions import Self - - from scrapy import Request, Spider - from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings - - -logger = logging.getLogger(__name__) - - -class AjaxCrawlMiddleware: - """ - Handle 'AJAX crawlable' pages marked as crawlable via meta tag. - """ - - def __init__(self, settings: BaseSettings): - if not settings.getbool("AJAXCRAWL_ENABLED"): - raise NotConfigured - - warn( - "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware is deprecated" - " and will be removed in a future Scrapy version.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - - # XXX: Google parses at least first 100k bytes; scrapy's redirect - # middleware parses first 4k. 4k turns out to be insufficient - # for this middleware, and parsing 100k could be slow. - # We use something in between (32K) by default. - self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE") - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings) - - def process_response( - self, request: Request, response: Response, spider: Spider - ) -> Request | Response: - if not isinstance(response, HtmlResponse) or response.status != 200: - return response - - if request.method != "GET": - # other HTTP methods are either not safe or don't have a body - return response - - if "ajax_crawlable" in request.meta: # prevent loops - return response - - if not self._has_ajax_crawlable_variant(response): - return response - - ajax_crawl_request = request.replace(url=escape_ajax(request.url + "#!")) - logger.debug( - "Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s", - {"ajax_crawl_request": ajax_crawl_request, "request": request}, - extra={"spider": spider}, - ) - - ajax_crawl_request.meta["ajax_crawlable"] = True - return ajax_crawl_request - - def _has_ajax_crawlable_variant(self, response: Response) -> bool: - """ - Return True if a page without hash fragment could be "AJAX crawlable". - """ - body = response.text[: self.lookup_bytes] - return _has_ajaxcrawlable_meta(body) - - -_ajax_crawlable_re: re.Pattern[str] = re.compile( - r'' -) - - -def _has_ajaxcrawlable_meta(text: str) -> bool: - """ - >>> _has_ajaxcrawlable_meta('') - True - >>> _has_ajaxcrawlable_meta("") - True - >>> _has_ajaxcrawlable_meta('') - False - >>> _has_ajaxcrawlable_meta('') - False - """ - - # Stripping scripts and comments is slow (about 20x slower than - # just checking if a string is in text); this is a quick fail-fast - # path that should work for most pages. - if "fragment" not in text: - return False - if "content" not in text: - return False - - text = html.remove_tags_with_content(text, ("script", "noscript")) - text = html.replace_entities(text) - text = html.remove_comments(text) - return _ajax_crawlable_re.search(text) is not None diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d945546d5..cd8c2abca 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -139,7 +139,7 @@ class CookiesMiddleware: for key in ("name", "value", "path", "domain"): value = cookie.get(key) if value is None: - if key in ("name", "value"): + if key in {"name", "value"}: msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" logger.warning(msg) return None @@ -176,7 +176,7 @@ class CookiesMiddleware: Extract cookies from the Request.cookies attribute """ if not request.cookies: - return [] + return () cookies: Iterable[VerboseCookie] if isinstance(request.cookies, dict): cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index c28c93d4e..b09abdadd 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -6,11 +6,14 @@ See documentation in docs/topics/downloader-middleware.rst from __future__ import annotations +import warnings from typing import TYPE_CHECKING from w3lib.http import basic_auth_header from scrapy import Request, Spider, signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.settings import SETTINGS_PRIORITIES from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.url import url_is_from_any_domain @@ -23,12 +26,28 @@ if TYPE_CHECKING: class HttpAuthMiddleware: - """Set Basic HTTP Authorization header - (http_user and http_pass spider class attributes)""" + """Set Basic HTTP Authorization header.""" + + def __init__(self) -> None: + self._auth: bytes | None = None + self._domain: str | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: o = cls() + usr = crawler.settings.get("HTTPAUTH_USER", "") + pwd = crawler.settings.get("HTTPAUTH_PASS", "") + if usr or pwd: + domain_priority = crawler.settings.getpriority("HTTPAUTH_DOMAIN") or 0 + if domain_priority <= SETTINGS_PRIORITIES["default"]: + raise ValueError( + "HTTPAUTH_DOMAIN must be set when HTTPAUTH_USER or HTTPAUTH_PASS " + "is configured. Set it to a domain (e.g. 'example.com') to restrict " + "credentials to that domain, or set it to None to send credentials " + "with all requests." + ) + o._auth = basic_auth_header(usr, pwd) + o._domain = crawler.settings.get("HTTPAUTH_DOMAIN") crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o @@ -36,18 +55,34 @@ class HttpAuthMiddleware: usr = getattr(spider, "http_user", "") pwd = getattr(spider, "http_pass", "") if usr or pwd: - self.auth = basic_auth_header(usr, pwd) - self.domain = spider.http_auth_domain # type: ignore[attr-defined] + warnings.warn( + "Use the HTTPAUTH_USER, HTTPAUTH_PASS, and HTTPAUTH_DOMAIN settings " + "instead of the http_user, http_pass, and http_auth_domain spider " + "attributes. Support for the spider attributes will be removed in a " + "future version of Scrapy.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._auth = basic_auth_header(usr, pwd) + self._domain = spider.http_auth_domain # type: ignore[attr-defined] @_warn_spider_arg def process_request( self, request: Request, spider: Spider | None = None ) -> Request | Response | None: - auth = getattr(self, "auth", None) - if ( - auth - and b"Authorization" not in request.headers - and (not self.domain or url_is_from_any_domain(request.url, [self.domain])) + if b"Authorization" in request.headers: + return None + # Per-request meta overrides + usr = request.meta.get("http_user", "") + pwd = request.meta.get("http_pass", "") + if usr or pwd: + domain = request.meta.get("http_auth_domain") + if not domain or url_is_from_any_domain(request.url, [domain]): + request.headers[b"Authorization"] = basic_auth_header(usr, pwd) + return None + # Middleware-level auth + if self._auth and ( + not self._domain or url_is_from_any_domain(request.url, [self._domain]) ): - request.headers[b"Authorization"] = auth + request.headers[b"Authorization"] = self._auth return None diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index a176dcd9f..c6c811809 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -107,7 +107,7 @@ class HttpCacheMiddleware: return response # Skip cached responses and uncacheable requests - if "cached" in response.flags or "_dont_cache" in request.meta: + if "_dont_cache" in request.meta or "cached" in response.flags: request.meta.pop("_dont_cache", None) return response diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index d4fa2d4d7..414c3d8a3 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -40,12 +40,13 @@ except ImportError: pass else: try: - brotli.Decompressor.can_accept_more_data + brotli.Decompressor.can_accept_more_data # noqa: B018 except AttributeError: # pragma: no cover warnings.warn( "You have brotli installed. But 'br' encoding support now requires " "brotli's or brotlicffi's version >= 1.2.0. Please upgrade " "brotli/brotlicffi to make Scrapy decode 'br' encoded responses.", + stacklevel=2, ) else: ACCEPTED_ENCODINGS.append(b"br") diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index d3d46a947..e034dced1 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -69,7 +69,7 @@ class HttpProxyMiddleware: _scheme = parsed.scheme if ( # 'no_proxy' is only supported by http schemes - _scheme not in ("http", "https") + _scheme not in {"http", "https"} or (parsed.hostname and not proxy_bypass(parsed.hostname)) ) and _scheme in self.proxies: scheme = _scheme diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 33d7ba609..e03dc040e 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -2,7 +2,6 @@ from __future__ import annotations import logging import re -import warnings from typing import TYPE_CHECKING from scrapy import Request, Spider, signals @@ -75,25 +74,10 @@ class OffsiteMiddleware: allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: return re.compile("") # allow all by default - url_pattern = re.compile(r"^https?://.*$") - port_pattern = re.compile(r":\d+$") domains = [] for domain in allowed_domains: if domain is None: continue - if url_pattern.match(domain): - message = ( - "allowed_domains accepts only domains, not URLs. " - f"Ignoring URL entry {domain} in allowed_domains." - ) - warnings.warn(message) - elif port_pattern.search(domain): - message = ( - "allowed_domains accepts only domains without ports. " - f"Ignoring entry {domain} in allowed_domains." - ) - warnings.warn(message) - else: - domains.append(re.escape(domain)) + domains.append(re.escape(domain)) regex = rf"^(.*\.)?({'|'.join(domains)})$" return re.compile(regex) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 61f62ecfc..821f41699 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -2,14 +2,17 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, cast -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response +from scrapy.spidermiddlewares.referer import RefererMiddleware from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import global_object_name from scrapy.utils.response import get_meta_refresh if TYPE_CHECKING: @@ -24,61 +27,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _build_redirect_request( - source_request: Request, *, url: str, **kwargs: Any -) -> Request: - redirect_request = source_request.replace( - url=url, - **kwargs, - cls=None, - cookies=None, - ) - if "_scheme_proxy" in redirect_request.meta: - source_request_scheme = urlparse_cached(source_request).scheme - redirect_request_scheme = urlparse_cached(redirect_request).scheme - if source_request_scheme != redirect_request_scheme: - redirect_request.meta.pop("_scheme_proxy") - redirect_request.meta.pop("proxy", None) - redirect_request.meta.pop("_auth_proxy", None) - redirect_request.headers.pop(b"Proxy-Authorization", None) - has_cookie_header = "Cookie" in redirect_request.headers - has_authorization_header = "Authorization" in redirect_request.headers - if has_cookie_header or has_authorization_header: - default_ports = {"http": 80, "https": 443} - - parsed_source_request = urlparse_cached(source_request) - source_scheme, source_host, source_port = ( - parsed_source_request.scheme, - parsed_source_request.hostname, - parsed_source_request.port - or default_ports.get(parsed_source_request.scheme), - ) - - parsed_redirect_request = urlparse_cached(redirect_request) - redirect_scheme, redirect_host, redirect_port = ( - parsed_redirect_request.scheme, - parsed_redirect_request.hostname, - parsed_redirect_request.port - or default_ports.get(parsed_redirect_request.scheme), - ) - - if has_cookie_header and ( - redirect_scheme not in {source_scheme, "https"} - or source_host != redirect_host - ): - del redirect_request.headers["Cookie"] - - # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name - if has_authorization_header and ( - source_scheme != redirect_scheme - or source_host != redirect_host - or source_port != redirect_port - ): - del redirect_request.headers["Authorization"] - - return redirect_request - - class BaseRedirectMiddleware: crawler: Crawler enabled_setting: str = "REDIRECT_ENABLED" @@ -89,13 +37,59 @@ class BaseRedirectMiddleware: self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES") self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST") + self._referer_spider_middleware: RefererMiddleware | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: o = cls(crawler.settings) o.crawler = crawler + crawler.signals.connect(o._engine_started, signal=signals.engine_started) return o + def handle_referer(self, request: Request, response: Response) -> None: + """Remove, modify or keep the Referer header of *request* based on the + *response* that triggered *request*. + + By default, this method finds a run-time instance of + scrapy.spidermiddlewares.referer.RefererMiddleware (or of a subclass) + and uses it to set the right Referer header. + + Override this method if you use a different Scrapy component to handle + Referer headers, of if you want to use a custom logic to set the + Referer header on redirects. + """ + request.headers.pop("Referer", None) + if not self._referer_spider_middleware: + return + self._referer_spider_middleware.get_processed_request(request, response) + + def _engine_started(self) -> None: + self._referer_spider_middleware = self.crawler.get_spider_middleware( + RefererMiddleware + ) + if self._referer_spider_middleware: + return + redirect_cls = global_object_name(self.__class__) + referer_cls = global_object_name(RefererMiddleware) + if self.__class__ in {RedirectMiddleware, MetaRefreshMiddleware}: + replacement = ( + f"replace {redirect_cls} with a subclass that overrides the " + f"handle_referer() method" + ) + else: + replacement = ( + f"or edit {redirect_cls} (if defined in your code base) to " + f"override the handle_referer() method, or replace " + f"{redirect_cls} with a subclass that overrides the " + f"handle_referer() method." + ) + logger.warning( + f"{redirect_cls} found no {referer_cls} instance to handle " + f"Referer header handling, so the Referer header will be removed " + f"on redirects. To set a Referer header on redirects, enable " + f"{referer_cls} (or a subclass), or {replacement}.", + ) + def _redirect(self, redirected: Request, request: Request, reason: Any) -> Request: ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times) redirects = request.meta.get("redirect_times", 0) + 1 @@ -126,17 +120,78 @@ class BaseRedirectMiddleware: ) raise IgnoreRequest("max redirections reached") - def _redirect_request_using_get( - self, request: Request, redirect_url: str + def _build_redirect_request( + self, source_request: Request, response: Response, *, url: str, **kwargs: Any ) -> Request: - redirect_request = _build_redirect_request( + redirect_request = source_request.replace( + url=url, + **kwargs, + cls=None, + cookies=None, + ) + if "_scheme_proxy" in redirect_request.meta: + source_request_scheme = urlparse_cached(source_request).scheme + redirect_request_scheme = urlparse_cached(redirect_request).scheme + if source_request_scheme != redirect_request_scheme: + redirect_request.meta.pop("_scheme_proxy") + redirect_request.meta.pop("proxy", None) + redirect_request.meta.pop("_auth_proxy", None) + redirect_request.headers.pop(b"Proxy-Authorization", None) + + has_cookie_header = "Cookie" in redirect_request.headers + has_authorization_header = "Authorization" in redirect_request.headers + if has_cookie_header or has_authorization_header: + default_ports = {"http": 80, "https": 443} + + parsed_source_request = urlparse_cached(source_request) + source_scheme, source_host, source_port = ( + parsed_source_request.scheme, + parsed_source_request.hostname, + parsed_source_request.port + or default_ports.get(parsed_source_request.scheme), + ) + + parsed_redirect_request = urlparse_cached(redirect_request) + redirect_scheme, redirect_host, redirect_port = ( + parsed_redirect_request.scheme, + parsed_redirect_request.hostname, + parsed_redirect_request.port + or default_ports.get(parsed_redirect_request.scheme), + ) + + if has_cookie_header and ( + redirect_scheme not in {source_scheme, "https"} + or source_host != redirect_host + ): + del redirect_request.headers["Cookie"] + + # https://fetch.spec.whatwg.org/#ref-for-cors-non-wildcard-request-header-name + if has_authorization_header and ( + source_scheme != redirect_scheme + or source_host != redirect_host + or source_port != redirect_port + ): + del redirect_request.headers["Authorization"] + + self.handle_referer(redirect_request, response) + + return redirect_request + + def _redirect_request_using_get( + self, request: Request, response: Response, redirect_url: str + ) -> Request: + redirect_request = self._build_redirect_request( request, + response, url=redirect_url, method="GET", body="", ) redirect_request.headers.pop("Content-Type", None) redirect_request.headers.pop("Content-Length", None) + redirect_request.headers.pop("Content-Encoding", None) + redirect_request.headers.pop("Content-Language", None) + redirect_request.headers.pop("Content-Location", None) return redirect_request @@ -153,14 +208,19 @@ class RedirectMiddleware(BaseRedirectMiddleware): if ( request.meta.get("dont_redirect", False) or response.status - in getattr(self.crawler.spider, "handle_httpstatus_list", []) - or response.status in request.meta.get("handle_httpstatus_list", []) + in getattr(self.crawler.spider, "handle_httpstatus_list", ()) + or response.status in request.meta.get("handle_httpstatus_list", ()) or request.meta.get("handle_httpstatus_all", False) ): return response - allowed_status = (301, 302, 303, 307, 308) - if "Location" not in response.headers or response.status not in allowed_status: + if "Location" not in response.headers or response.status not in { + 301, + 302, + 303, + 307, + 308, + }: return response assert response.headers["Location"] is not None @@ -170,14 +230,23 @@ class RedirectMiddleware(BaseRedirectMiddleware): location = request_scheme + "://" + location.lstrip("/") redirected_url = urljoin(request.url, location) - redirected = _build_redirect_request(request, url=redirected_url) + + if not urlparse(redirected_url).fragment: + fragment = urlparse_cached(request).fragment + if fragment: + redirected_url = urljoin(redirected_url, f"#{fragment}") + + redirected = self._build_redirect_request(request, response, url=redirected_url) if urlparse_cached(redirected).scheme not in {"http", "https"}: return response - if response.status in (301, 307, 308) or request.method == "HEAD": - return self._redirect(redirected, request, response.status) + if (response.status in {301, 302} and request.method == "POST") or ( + response.status == 303 and request.method not in {"GET", "HEAD"} + ): + redirected = self._redirect_request_using_get( + request, response, redirected_url + ) - redirected = self._redirect_request_using_get(request, redirected_url) return self._redirect(redirected, request, response.status) @@ -204,7 +273,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): interval, url = get_meta_refresh(response, ignore_tags=self._ignore_tags) if not url: return response - redirected = self._redirect_request_using_get(request, url) + redirected = self._redirect_request_using_get(request, response, url) if urlparse_cached(redirected).scheme not in {"http", "https"}: return response if cast("float", interval) < self._maxdelay: diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index b6dba5773..5f125cae4 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -12,7 +12,7 @@ once the spider has finished crawling all regular (non-failed) pages. from __future__ import annotations -from logging import Logger, getLogger +from logging import Logger, getLevelName, getLogger from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured @@ -25,11 +25,11 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + import scrapy from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.http.request import Request from scrapy.settings import BaseSettings - from scrapy.spiders import Spider retry_logger = getLogger(__name__) @@ -38,11 +38,12 @@ retry_logger = getLogger(__name__) def get_retry_request( request: Request, *, - spider: Spider, + spider: scrapy.Spider, reason: str | Exception | type[Exception] = "unspecified", max_retry_times: int | None = None, priority_adjust: int | None = None, logger: Logger = retry_logger, + give_up_log_level: int | str | None = None, stats_base_key: str = "retry", ) -> Request | None: """ @@ -51,14 +52,16 @@ def get_retry_request( exhausted. For example, in a :class:`~scrapy.Spider` callback, you could use it as - follows:: + follows: + + .. code-block:: python def parse(self, response): if not response.text: new_request_or_none = get_retry_request( response.request, spider=self, - reason='empty', + reason="empty", ) return new_request_or_none @@ -82,6 +85,10 @@ def get_retry_request( *logger* is the logging.Logger object to be used when logging messages + *give_up_log_level* is the :ref:`logging level ` used for the + message logged when a request exceeds its retries. See + :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details. + *stats_base_key* is a string to be used as the base key for the retry-related job stats """ @@ -114,8 +121,16 @@ def get_retry_request( stats.inc_value(f"{stats_base_key}/count") stats.inc_value(f"{stats_base_key}/reason_count/{reason}") return new_request + if give_up_log_level is None: + give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"] + if isinstance(give_up_log_level, str): + level = getLevelName(give_up_log_level) + if not isinstance(level, int): + raise ValueError(f"Invalid give-up log level: {give_up_log_level!r}") + give_up_log_level = level stats.inc_value(f"{stats_base_key}/max_reached") - logger.error( + logger.log( + give_up_log_level, "Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", {"request": request, "retry_times": retry_times, "reason": reason}, extra={"spider": spider}, @@ -132,6 +147,7 @@ class RetryMiddleware: self.max_retry_times = settings.getint("RETRY_TIMES") self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") + self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"] self.exceptions_to_retry = tuple( load_object(x) if isinstance(x, str) else x for x in settings.getlist("RETRY_EXCEPTIONS") @@ -145,7 +161,10 @@ class RetryMiddleware: @_warn_spider_arg def process_response( - self, request: Request, response: Response, spider: Spider | None = None + self, + request: Request, + response: Response, + spider: scrapy.Spider | None = None, ) -> Request | Response: if request.meta.get("dont_retry", False): return response @@ -156,7 +175,10 @@ class RetryMiddleware: @_warn_spider_arg def process_exception( - self, request: Request, exception: Exception, spider: Spider | None = None + self, + request: Request, + exception: Exception, + spider: scrapy.Spider | None = None, ) -> Request | Response | None: if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False @@ -169,6 +191,9 @@ class RetryMiddleware: ) -> Request | None: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) + give_up_log_level = request.meta.get( + "give_up_log_level", self.give_up_log_level + ) assert self.crawler.spider return get_retry_request( request, @@ -176,4 +201,5 @@ class RetryMiddleware: spider=self.crawler.spider, max_retry_times=max_retry_times, priority_adjust=priority_adjust, + give_up_log_level=give_up_log_level, ) diff --git a/scrapy/extension.py b/scrapy/extension.py index 9f978fa32..05cce326b 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -20,4 +20,6 @@ class ExtensionManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: - return build_component_list(settings.getwithbase("EXTENSIONS")) + return build_component_list( + settings.get_component_priority_dict_with_base("EXTENSIONS") + ) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 1400f4c9b..6a5e55992 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -5,6 +5,7 @@ Extension for collecting core stats like items scraped and start/finish times from __future__ import annotations from datetime import datetime, timezone +from time import monotonic from typing import TYPE_CHECKING, Any from scrapy import Spider, signals @@ -21,6 +22,7 @@ class CoreStats: def __init__(self, stats: StatsCollector): self.stats: StatsCollector = stats self.start_time: datetime | None = None + self._start_time_mono: float | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -35,13 +37,14 @@ class CoreStats: def spider_opened(self, spider: Spider) -> None: self.start_time = datetime.now(tz=timezone.utc) + self._start_time_mono = monotonic() self.stats.set_value("start_time", self.start_time) def spider_closed(self, spider: Spider, reason: str) -> None: assert self.start_time is not None - finish_time = datetime.now(tz=timezone.utc) - elapsed_time = finish_time - self.start_time - elapsed_time_seconds = elapsed_time.total_seconds() + assert self._start_time_mono is not None + finish_time, finish_time_mono = datetime.now(tz=timezone.utc), monotonic() + elapsed_time_seconds = finish_time_mono - self._start_time_mono self.stats.set_value("elapsed_time_seconds", elapsed_time_seconds) self.stats.set_value("finish_time", finish_time) self.stats.set_value("finish_reason", reason) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 86593aba8..8029f85c9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -21,14 +21,13 @@ from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, cast from urllib.parse import unquote, urlparse from twisted.internet.defer import Deferred, DeferredList -from twisted.internet.threads import deferToThread from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.asyncio import is_asyncio_available, run_in_thread from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file @@ -131,7 +130,7 @@ class BlockingFeedStorage(ABC): return NamedTemporaryFile(prefix="feed-", dir=path) def store(self, file: IO[bytes]) -> Deferred[None] | None: - return deferToThread(self._store_in_thread, file) + return deferred_from_coro(run_in_thread(self._store_in_thread, file)) @abstractmethod def _store_in_thread(self, file: IO[bytes]) -> None: @@ -168,7 +167,7 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None): - self.path: str = file_uri_to_path(uri) if uri.startswith("file://") else uri + self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri feed_options = feed_options or {} self.write_mode: OpenBinaryMode = ( "wb" if feed_options.get("overwrite", False) else "ab" @@ -201,7 +200,7 @@ class S3FeedStorage(BlockingFeedStorage): try: import boto3.session # noqa: PLC0415 except ImportError: - raise NotConfigured("missing boto3 library") + raise NotConfigured("missing boto3 library") from None u = urlparse(uri) assert u.hostname self.bucketname: str = u.hostname @@ -251,10 +250,19 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - kwargs: dict[str, Any] = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} - self.s3_client.upload_fileobj( - Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs - ) + if self.acl: + self.s3_client.upload_fileobj( + Bucket=self.bucketname, + Key=self.keyname, + Fileobj=file, + ExtraArgs={"ACL": self.acl}, + ) + else: + self.s3_client.upload_fileobj( + Bucket=self.bucketname, + Key=self.keyname, + Fileobj=file, + ) file.close() @@ -298,12 +306,15 @@ class GCSFeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - from google.cloud.storage import Client # noqa: PLC0415 + try: + from google.cloud.storage import Client # noqa: PLC0415 - client = Client(project=self.project_id) - bucket = client.get_bucket(self.bucket_name) - blob = bucket.blob(self.blob_name) - blob.upload_from_file(file, predefined_acl=self.acl) + client = Client(project=self.project_id) + bucket = client.get_bucket(self.bucket_name) + blob = bucket.blob(self.blob_name) + blob.upload_from_file(file, predefined_acl=self.acl) + finally: + file.close() class FTPFeedStorage(BlockingFeedStorage): @@ -469,9 +480,13 @@ class FeedExporter: # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' - for uri, feed_options in self.settings.getdict("FEEDS").items(): + for settings_uri, feed_options in self.settings.getdict("FEEDS").items(): # handle pathlib.Path objects - uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() + uri = ( + str(settings_uri) + if not isinstance(settings_uri, Path) + else settings_uri.absolute().as_uri() + ) self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings ) @@ -522,13 +537,15 @@ class FeedExporter: # Send FEED_EXPORTER_CLOSED signal await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed) + @staticmethod + def _get_file(slot_: FeedSlot) -> IO[bytes]: + assert slot_.file + if isinstance(slot_.file, PostProcessingManager): + slot_.file.close() + return slot_.file.file + return slot_.file + async def _close_slot(self, slot: FeedSlot, spider: Spider) -> None: - def get_file(slot_: FeedSlot) -> IO[bytes]: - assert slot_.file - if isinstance(slot_.file, PostProcessingManager): - slot_.file.close() - return slot_.file.file - return slot_.file if slot.itemcount: # Normal case @@ -545,7 +562,7 @@ class FeedExporter: slot_type = type(slot.storage).__name__ assert self.crawler.stats try: - await ensure_awaitable(slot.storage.store(get_file(slot))) + await ensure_awaitable(slot.storage.store(self._get_file(slot))) except Exception: logger.error( "Error storing %s", @@ -693,9 +710,7 @@ class FeedExporter: uri_params_function: str | UriParamsCallableT | None, slot: FeedSlot | None = None, ) -> dict[str, Any]: - params = {} - for k in dir(spider): - params[k] = getattr(spider, k) + params = {k: getattr(spider, k) for k in dir(spider)} utc_now = datetime.now(tz=timezone.utc) params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index efda50a82..dbb79b02d 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -106,10 +106,10 @@ class RFC2616Policy: if b"max-age" in cc or b"Expires" in response.headers: return True # Firefox fallbacks this statuses to one year expiration if none is set - if response.status in (300, 301, 308): + if response.status in {300, 301, 308}: return True # Other statuses without expiration requires at least one validator - if response.status in (200, 203, 401): + if response.status in {200, 203, 401}: return b"Last-Modified" in response.headers or b"ETag" in response.headers # Any other is probably not eligible for caching # Makes no sense to cache responses that does not contain expiration @@ -216,7 +216,7 @@ class RFC2616Policy: return (date - lastmodified) / 10 # This request can be cached indefinitely - if response.status in (300, 301, 308): + if response.status in {300, 301, 308}: return self.MAXAGE # Insufficient information to compute freshness lifetime @@ -313,7 +313,9 @@ class FilesystemCacheStorage: self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") # https://github.com/python/mypy/issues/10740 - self._open: Callable[Concatenate[str | os.PathLike, str, ...], IO[bytes]] = ( + self._open: Callable[ + Concatenate[str | os.PathLike[str], str, ...], IO[bytes] + ] = ( gzip.open if self.use_gzip else open # type: ignore[assignment] ) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 990c144b5..1444c8941 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -9,13 +9,13 @@ from __future__ import annotations import logging import socket import sys +import warnings from importlib import import_module from pprint import pformat from typing import TYPE_CHECKING from scrapy import signals -from scrapy.exceptions import NotConfigured -from scrapy.mail import MailSender +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.defer import _schedule_coro from scrapy.utils.engine import get_engine_status @@ -39,18 +39,29 @@ class MemoryUsage: try: # stdlib's resource module is only available on unix platforms. self.resource = import_module("resource") - except ImportError: - raise NotConfigured + except ImportError as exc: + raise NotConfigured from exc self.crawler: Crawler = crawler self.warned: bool = False self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + if self.notify_mails: # pragma: no cover + from scrapy.mail import MailSender # noqa: PLC0415 + + warnings.warn( + "The 'MEMUSAGE_NOTIFY_MAIL' setting is deprecated and will be removed " + "in a future release. Please use the 'memusage_warning_reached' and 'spider_closed' " + "signals to implement custom notifications.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self.mail = MailSender.from_crawler(crawler) + self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 self.check_interval: float = crawler.settings.getfloat( "MEMUSAGE_CHECK_INTERVAL_SECONDS" ) - self.mail: MailSender = MailSender.from_crawler(crawler) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @@ -102,7 +113,7 @@ class MemoryUsage: {"memusage": mem}, extra={"crawler": self.crawler}, ) - if self.notify_mails: + if self.notify_mails: # pragma: no cover subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " f"memory usage exceeded {mem}MiB at {socket.gethostname()}" @@ -128,13 +139,14 @@ class MemoryUsage: assert self.crawler.stats if self.get_virtual_size() > self.warning: self.crawler.stats.set_value("memusage/warning_reached", 1) + self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached) mem = self.warning / 1024 / 1024 logger.warning( "Memory usage reached %(memusage)dMiB", {"memusage": mem}, extra={"crawler": self.crawler}, ) - if self.notify_mails: + if self.notify_mails: # pragma: no cover subj = ( f"{self.crawler.settings['BOT_NAME']} warning: " f"memory usage reached {mem}MiB at {socket.gethostname()}" @@ -143,7 +155,7 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts: list[str], subject: str) -> None: + def _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover """send notification mail with some additional useful info""" assert self.crawler.engine assert self.crawler.stats diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 860b97a55..cd35c8165 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -10,6 +10,7 @@ from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: + from collections.abc import Sequence from json import JSONEncoder from twisted.internet.task import LoopingCall @@ -31,8 +32,8 @@ class PeriodicLog: self, stats: StatsCollector, interval: float = 60.0, - ext_stats: dict[str, Any] = {}, - ext_delta: dict[str, Any] = {}, + ext_stats: dict[str, Any] | None = None, + ext_delta: dict[str, Any] | None = None, ext_timing_enabled: bool = False, ): self.stats: StatsCollector = stats @@ -41,11 +42,19 @@ class PeriodicLog: self.task: AsyncioLoopingCall | LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) - self.ext_stats_include: list[str] = ext_stats.get("include", []) - self.ext_stats_exclude: list[str] = ext_stats.get("exclude", []) + self.ext_stats_include: Sequence[str] = ( + ext_stats.get("include", ()) if ext_stats else () + ) + self.ext_stats_exclude: Sequence[str] = ( + ext_stats.get("exclude", ()) if ext_stats else () + ) self.ext_delta_enabled: bool = bool(ext_delta) - self.ext_delta_include: list[str] = ext_delta.get("include", []) - self.ext_delta_exclude: list[str] = ext_delta.get("exclude", []) + self.ext_delta_include: Sequence[str] = ( + ext_delta.get("include", ()) if ext_delta else () + ) + self.ext_delta_exclude: Sequence[str] = ( + ext_delta.get("exclude", ()) if ext_delta else () + ) self.ext_timing_enabled: bool = ext_timing_enabled @classmethod @@ -143,7 +152,7 @@ class PeriodicLog: return {"stats": stats} def param_allowed( - self, stat_name: str, include: list[str], exclude: list[str] + self, stat_name: str, include: Sequence[str], exclude: Sequence[str] ) -> bool: if not include and not exclude: return True diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 9e900308d..f05595806 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -6,10 +6,11 @@ Use STATSMAILER_RCPTS setting to enable and give the recipient mail address from __future__ import annotations +import warnings from typing import TYPE_CHECKING from scrapy import Spider, signals -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.mail import MailSender if TYPE_CHECKING: @@ -21,6 +22,13 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.statscollectors import StatsCollector +warnings.warn( + "The scrapy.extensions.statsmailer module is deprecated and will be " + "removed in a future release.", + stacklevel=2, + category=ScrapyDeprecationWarning, +) + class StatsMailer: def __init__(self, stats: StatsCollector, recipients: list[str], mail: MailSender): diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index fec6b2d56..3be24c53f 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -44,7 +44,7 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured - if not crawler.settings.getbool("TWISTED_ENABLED"): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): raise NotConfigured( "The TelnetConsole extension requires a Twisted reactor." " You can set the TELNETCONSOLE_ENABLED setting to False to remove this warning." diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index 0e5c2b53b..e20e894ae 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -5,9 +5,11 @@ Use this module (instead of the more specific ones) when importing Headers, Request and Response outside this module. """ +from warnings import catch_warnings, filterwarnings + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.headers import Headers from scrapy.http.request import Request -from scrapy.http.request.form import FormRequest from scrapy.http.request.json_request import JsonRequest from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.response import Response @@ -15,6 +17,19 @@ from scrapy.http.response.html import HtmlResponse from scrapy.http.response.json import JsonResponse from scrapy.http.response.text import TextResponse from scrapy.http.response.xml import XmlResponse +from scrapy.utils.deprecate import create_deprecated_class + +with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + + from scrapy.http.request.form import FormRequest as _FormRequest + + FormRequest = create_deprecated_class( + name="FormRequest", + new_class=_FormRequest, + subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.", + instance_warn_message="{cls} is deprecated, use the form2request library instead.", + ) __all__ = [ "FormRequest", diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 13d1c85d0..599f20947 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -54,14 +54,14 @@ class CookieJar: if not IPV4_RE.search(req_host): hosts = potential_domain_matches(req_host) if "." not in req_host: - hosts += [req_host + ".local"] + hosts.append(req_host + ".local") else: hosts = [req_host] cookies = [] for host in hosts: if host in self.jar._cookies: # type: ignore[attr-defined] - cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined] + cookies.extend(self.jar._cookies_for_domain(host, wreq)) # type: ignore[attr-defined] attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined] if attrs and not wreq.has_header("Cookie"): @@ -74,7 +74,7 @@ class CookieJar: @property def _cookies(self) -> dict[str, dict[str, dict[str, Cookie]]]: - return self.jar._cookies # type: ignore[attr-defined] + return self.jar._cookies # type: ignore[attr-defined,no-any-return] def clear_session_cookies(self) -> None: return self.jar.clear_session_cookies() diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index c4a0c8131..7db648a45 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -138,6 +138,7 @@ class Request(object_ref): ) -> None: self._encoding: str = encoding # this one has to be set first self.method: str = str(method).upper() + self._meta: dict[str, Any] | None = dict(meta) if meta else None self._set_url(url) self._set_body(body) if not isinstance(priority, int): @@ -232,7 +233,6 @@ class Request(object_ref): #: default. See :meth:`~scrapy.Spider.start`. self.dont_filter: bool = dont_filter - self._meta: dict[str, Any] | None = dict(meta) if meta else None self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None self._flags: list[str] | None = list(flags) if flags else None @@ -242,29 +242,27 @@ class Request(object_ref): self._cb_kwargs = {} return self._cb_kwargs - @cb_kwargs.setter - def cb_kwargs(self, value: dict[str, Any] | None) -> None: - self._cb_kwargs = value or None - @property def meta(self) -> dict[str, Any]: if self._meta is None: self._meta = {} return self._meta - @meta.setter - def meta(self, value: dict[str, Any] | None) -> None: - self._meta = value or None - @property def url(self) -> str: return self._url + def _url_is_verbatim(self) -> bool: + return bool(self._meta and self._meta.get("verbatim_url")) + def _set_url(self, url: str) -> None: if not isinstance(url, str): raise TypeError(f"Request url must be str, got {type(url).__name__}") - self._url = safe_url_string(url, self.encoding) + if self._url_is_verbatim(): + self._url = url + else: + self._url = safe_url_string(url, self.encoding) if ( "://" not in self._url @@ -292,7 +290,7 @@ class Request(object_ref): @flags.setter def flags(self, value: list[str] | None) -> None: - self._flags = value or None + self._flags = value @property def cookies(self) -> CookiesT: @@ -302,7 +300,7 @@ class Request(object_ref): @cookies.setter def cookies(self, value: CookiesT | None) -> None: - self._cookies = value or None + self._cookies = value @property def headers(self) -> Headers: @@ -317,7 +315,9 @@ class Request(object_ref): if isinstance(value, Headers): self._headers = value else: - self._headers = Headers(value, encoding=self.encoding) if value else None + self._headers = ( + Headers(value, encoding=self.encoding) if value is not None else None + ) def __repr__(self) -> str: return f"<{self.method} {self.url}>" diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d2d13b8df..4da595b22 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -8,12 +8,14 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, TypeAlias, cast +from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit +from warnings import warn from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.request import Request from scrapy.utils.python import is_listlike, to_bytes @@ -30,6 +32,12 @@ if TYPE_CHECKING: from scrapy.http.response.text import TextResponse +warn( + "The entire scrapy.http.request.form module is deprecated. Use the " + "form2request library instead.", + ScrapyDeprecationWarning, + stacklevel=2, +) FormdataVType: TypeAlias = str | Iterable[str] FormdataKVType: TypeAlias = tuple[str, FormdataVType] @@ -39,7 +47,7 @@ FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None class FormRequest(Request): __slots__ = () - valid_form_methods = ["GET", "POST"] + valid_form_methods: ClassVar[list[str]] = ["GET", "POST"] def __init__( self, *args: Any, formdata: FormdataType = None, **kwargs: Any @@ -153,7 +161,7 @@ def _get_form( try: form = forms[formnumber] except IndexError: - raise IndexError(f"Form number {formnumber} not found in {response}") + raise IndexError(f"Form number {formnumber} not found in {response}") from None return cast("FormElement", form) @@ -167,7 +175,7 @@ def _get_inputs( try: formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): - raise ValueError("formdata should be a dict or iterable of tuples") + raise ValueError("formdata should be a dict or iterable of tuples") from None if not formdata: formdata = [] diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1776bdca8..13fee06d8 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -36,7 +36,9 @@ class JsonRequest(Request): data_passed: bool = data is not None if body_passed and data_passed: - warnings.warn("Both body and data passed. data will be ignored") + warnings.warn( + "Both body and data passed. data will be ignored", stacklevel=2 + ) elif not body_passed and data_passed: kwargs["body"] = self._dumps(data) if "method" not in kwargs: @@ -68,7 +70,9 @@ class JsonRequest(Request): data_passed: bool = data is not None if body_passed and data_passed: - warnings.warn("Both body and data passed. data will be ignored") + warnings.warn( + "Both body and data passed. data will be ignored", stacklevel=2 + ) elif not body_passed and data_passed: kwargs["body"] = self._dumps(data) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 2cc35fef4..09b1c8b32 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -20,7 +20,6 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterable, Mapping from ipaddress import IPv4Address, IPv6Address - from twisted.internet.ssl import Certificate from twisted.python.failure import Failure # typing.Self requires Python 3.11 @@ -38,17 +37,20 @@ class Response(object_ref): downloaded (by the Downloader) and fed to the Spiders for processing. """ - attributes: tuple[str, ...] = ( - "url", + __attrs_and_slots = ( "status", - "headers", - "body", - "flags", "request", "certificate", "ip_address", "protocol", ) + attributes: tuple[str, ...] = ( + "url", + "headers", + "body", + "flags", + *__attrs_and_slots, + ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the ``__init__()`` method. @@ -56,6 +58,16 @@ class Response(object_ref): Currently used by :meth:`Response.replace`. """ + __slots__ = ( + "__weakref__", + "_url", + "_body", + "_headers", + "_flags", + *__attrs_and_slots, + ) + del __attrs_and_slots + def __init__( self, url: str, @@ -64,17 +76,17 @@ class Response(object_ref): body: bytes = b"", flags: list[str] | None = None, request: Request | None = None, - certificate: Certificate | None = None, + certificate: Any = None, ip_address: IPv4Address | IPv6Address | None = None, protocol: str | None = None, ): - self.headers: Headers = Headers(headers or {}) + self._headers: Headers | None = Headers(headers) if headers else None self.status: int = int(status) self._set_body(body) self._set_url(url) self.request: Request | None = request - self.flags: list[str] = [] if flags is None else list(flags) - self.certificate: Certificate | None = certificate + self._flags: list[str] | None = list(flags) if flags else None + self.certificate: Any = certificate self.ip_address: IPv4Address | IPv6Address | None = ip_address self.protocol: str | None = protocol @@ -86,7 +98,7 @@ class Response(object_ref): raise AttributeError( "Response.cb_kwargs not available, this response " "is not tied to any request" - ) + ) from None @property def meta(self) -> dict[str, Any]: @@ -95,7 +107,7 @@ class Response(object_ref): except AttributeError: raise AttributeError( "Response.meta not available, this response is not tied to any request" - ) + ) from None @property def url(self) -> str: @@ -125,6 +137,31 @@ class Response(object_ref): else: self._body = body + @property + def headers(self) -> Headers: + if self._headers is None: + self._headers = Headers() + return self._headers + + @headers.setter + def headers( + self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + ) -> None: + if isinstance(value, Headers): + self._headers = value + else: + self._headers = Headers(value) if value is not None else None + + @property + def flags(self) -> list[str]: + if self._flags is None: + self._flags = [] + return self._flags + + @flags.setter + def flags(self, value: list[str] | None) -> None: + self._flags = value + def __repr__(self) -> str: return f"<{self.status} {self.url}>" diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index 7eed052c2..70c08c11d 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse class HtmlResponse(TextResponse): - pass + __slots__ = () diff --git a/scrapy/http/response/json.py b/scrapy/http/response/json.py index 219691094..0428dde6e 100644 --- a/scrapy/http/response/json.py +++ b/scrapy/http/response/json.py @@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse class JsonResponse(TextResponse): - pass + __slots__ = () diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 012ead519..6876e35e8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -41,15 +41,22 @@ _NONE = object() class TextResponse(Response): _DEFAULT_ENCODING = "ascii" - _cached_decoded_json = _NONE attributes: tuple[str, ...] = (*Response.attributes, "encoding") + __slots__ = ( + "_cached_benc", + "_cached_decoded_json", + "_cached_selector", + "_cached_ubody", + "_encoding", + ) def __init__(self, *args: Any, **kwargs: Any): self._encoding: str | None = kwargs.pop("encoding", None) self._cached_benc: str | None = None self._cached_ubody: str | None = None self._cached_selector: Selector | None = None + self._cached_decoded_json: object = _NONE super().__init__(*args, **kwargs) def _set_body(self, body: str | bytes | None) -> None: @@ -100,7 +107,7 @@ class TextResponse(Response): @memoizemethod_noargs def _headers_encoding(self) -> str | None: - content_type = cast("bytes", self.headers.get(b"Content-Type", b"")) + content_type = self.headers.get(b"Content-Type") or b"" return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) def _body_inferred_encoding(self) -> str: @@ -138,10 +145,10 @@ class TextResponse(Response): @property def selector(self) -> Selector: - # circular import - from scrapy.selector import Selector # noqa: PLC0415 - if self._cached_selector is None: + # circular import + from scrapy.selector import Selector # noqa: PLC0415 + self._cached_selector = Selector(self) return self._cached_selector @@ -213,7 +220,7 @@ class TextResponse(Response): def follow_all( self, - urls: Iterable[str | Link] | parsel.SelectorList | None = None, + urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None, callback: CallbackT | None = None, method: str = "GET", headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, @@ -297,7 +304,7 @@ def _url_from_selector(sel: parsel.Selector) -> str: return strip_html5_whitespace(sel.root) if not hasattr(sel.root, "tag"): raise _InvalidSelector(f"Unsupported selector: {sel}") - if sel.root.tag not in ("a", "link"): + if sel.root.tag not in {"a", "link"}: raise _InvalidSelector( f"Only and elements are supported; got <{sel.root.tag}>" ) diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index abf474a2f..6d9c4cb73 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse class XmlResponse(TextResponse): - pass + __slots__ = () diff --git a/scrapy/link.py b/scrapy/link.py index 9c272ab2f..046630403 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -39,7 +39,7 @@ class Link: def __eq__(self, other: object) -> bool: if not isinstance(other, Link): - raise NotImplementedError + return NotImplemented return ( self.url == other.url and self.text == other.text diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 96a0f523b..11fd62fb2 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -114,8 +114,8 @@ class LxmlParserLinkExtractor: # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) try: if self.strip: - attr_val = strip_html5_whitespace(attr_val) - attr_val = urljoin(base_url, attr_val) + attr_val = strip_html5_whitespace(attr_val) # noqa: PLW2901 this is intended + attr_val = urljoin(base_url, attr_val) # noqa: PLW2901 except ValueError: continue # skipping bogus links else: @@ -245,7 +245,7 @@ class LxmlLinkExtractor: if self.allow_res else [True] ) - denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] + denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else () return any(allowed) and not any(denied) def _process_links(self, links: list[Link]) -> list[Link]: diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 2f5c0343b..8ee7dc16c 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -27,7 +27,7 @@ class ItemLoader(itemloaders.ItemLoader): :param item: The item instance to populate using subsequent calls to :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`, or :meth:`~ItemLoader.add_value`. - :type item: scrapy.item.Item + :type item: :ref:`item object ` :param selector: The selector to extract data from, when using the :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or diff --git a/scrapy/mail.py b/scrapy/mail.py index 84209effa..fbd11ad1d 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -7,6 +7,7 @@ See documentation in docs/topics/email.rst from __future__ import annotations import logging +import warnings from email import encoders as Encoders from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart @@ -19,8 +20,10 @@ from typing import IO, TYPE_CHECKING, Any from twisted.internet import ssl from twisted.internet.defer import Deferred +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes +from scrapy.utils.reactorless import is_reactorless if TYPE_CHECKING: from collections.abc import Callable, Sequence @@ -49,6 +52,14 @@ def _to_bytes_or_none(text: str | bytes | None) -> bytes | None: return to_bytes(text) +warnings.warn( + "The scrapy.mail module is deprecated and will be removed in a future release. " + "Please use a dedicated Python mail library instead.", + stacklevel=2, + category=ScrapyDeprecationWarning, +) + + class MailSender: def __init__( self, @@ -61,6 +72,8 @@ class MailSender: smtpssl: bool = False, debug: bool = False, ): + if is_reactorless(): # pragma: no cover + raise RuntimeError(f"{type(self).__name__} requires a Twisted reactor.") self.smtphost: str = smtphost self.smtpport: int = smtpport self.smtpuser: bytes | None = _to_bytes_or_none(smtpuser) @@ -71,7 +84,7 @@ class MailSender: self.debug: bool = debug @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: + def from_crawler(cls, crawler: Crawler) -> Self: # pragma: no cover settings = crawler.settings return cls( smtphost=settings["MAIL_HOST"], diff --git a/scrapy/middleware.py b/scrapy/middleware.py index c4e0ce16e..37ec0f583 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -50,11 +50,8 @@ class MiddlewareManager(ABC): ) self.middlewares: tuple[Any, ...] = middlewares # Only process_spider_output and process_spider_exception can be None. - # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = ( - defaultdict(deque) - ) - self._mw_methods_requiring_spider: set[Callable] = set() + self.methods: dict[str, deque[Callable[..., Any] | None]] = defaultdict(deque) + self._mw_methods_requiring_spider: set[Callable[..., Any]] = set() for mw in middlewares: self._add_middleware(mw) @@ -119,7 +116,7 @@ class MiddlewareManager(ABC): def _add_middleware(self, mw: Any) -> None: # noqa: B027 pass - def _check_mw_method_spider_arg(self, method: Callable) -> None: + def _check_mw_method_spider_arg(self, method: Callable[..., Any]) -> None: if argument_is_required(method, "spider"): warnings.warn( f"{method.__qualname__}() requires a spider argument," diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index f58864471..84fb5f85e 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -33,18 +33,20 @@ class ItemPipelineManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: - return build_component_list(settings.getwithbase("ITEM_PIPELINES")) + return build_component_list( + settings.get_component_priority_dict_with_base("ITEM_PIPELINES") + ) - def _add_middleware(self, pipe: Any) -> None: - if hasattr(pipe, "open_spider"): - self.methods["open_spider"].append(pipe.open_spider) - self._check_mw_method_spider_arg(pipe.open_spider) - if hasattr(pipe, "close_spider"): - self.methods["close_spider"].appendleft(pipe.close_spider) - self._check_mw_method_spider_arg(pipe.close_spider) - if hasattr(pipe, "process_item"): - self.methods["process_item"].append(pipe.process_item) - self._check_mw_method_spider_arg(pipe.process_item) + def _add_middleware(self, mw: Any) -> None: + if hasattr(mw, "open_spider"): + self.methods["open_spider"].append(mw.open_spider) + self._check_mw_method_spider_arg(mw.open_spider) + if hasattr(mw, "close_spider"): + self.methods["close_spider"].appendleft(mw.close_spider) + self._check_mw_method_spider_arg(mw.close_spider) + if hasattr(mw, "process_item"): + self.methods["process_item"].append(mw.process_item) + self._check_mw_method_spider_arg(mw.process_item) def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: warnings.warn( @@ -60,32 +62,44 @@ class ItemPipelineManager(MiddlewareManager): "process_item", item, add_spider=True, warn_deferred=True ) + def _get_dfd( + self, + method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None], + ) -> Deferred[None]: + if method in self._mw_methods_requiring_spider: + return _maybeDeferred_coro(method, True, self._spider) + return _maybeDeferred_coro(method, True) + + @staticmethod + def _eb(failure: Failure) -> Failure: + assert isinstance(failure.value, FirstError) + return failure.value.subFailure + def _process_parallel_dfd(self, methodname: str) -> Deferred[list[None]]: methods = cast( "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]", self.methods[methodname], ) - def get_dfd( - method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None], - ) -> Deferred[None]: - if method in self._mw_methods_requiring_spider: - return _maybeDeferred_coro(method, True, self._spider) - return _maybeDeferred_coro(method, True) - - dfds = [get_dfd(m) for m in methods] + dfds = [self._get_dfd(m) for m in methods] d: Deferred[list[tuple[bool, None]]] = DeferredList( dfds, fireOnOneErrback=True, consumeErrors=True ) d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r]) - def eb(failure: Failure) -> Failure: - assert isinstance(failure.value, FirstError) - return failure.value.subFailure - - d2.addErrback(eb) + d2.addErrback(self._eb) return d2 + def get_awaitable( + self, + method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None], + ) -> Awaitable[None]: + if method in self._mw_methods_requiring_spider: + result = method(self._spider) + else: + result = method() + return ensure_awaitable(result, _warn=global_object_name(method)) + async def _process_parallel_asyncio(self, methodname: str) -> list[None]: methods = cast( "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]", @@ -94,16 +108,7 @@ class ItemPipelineManager(MiddlewareManager): if not methods: return [] - def get_awaitable( - method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None], - ) -> Awaitable[None]: - if method in self._mw_methods_requiring_spider: - result = method(self._spider) - else: - result = method() - return ensure_awaitable(result, _warn=global_object_name(method)) - - awaitables = [get_awaitable(m) for m in methods] + awaitables = [self.get_awaitable(m) for m in methods] await asyncio.gather(*awaitables) return [None for _ in methods] diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 55e01c4ab..0066fd38f 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -18,25 +18,28 @@ from contextlib import suppress from ftplib import FTP from io import BytesIO from pathlib import Path -from typing import IO, TYPE_CHECKING, Any, NoReturn, Protocol, TypedDict, cast +from typing import IO, TYPE_CHECKING, Any, ClassVar, NoReturn, Protocol, TypedDict, cast from urllib.parse import urlparse from itemadapter import ItemAdapter from twisted.internet.defer import Deferred, maybeDeferred -from twisted.internet.threads import deferToThread from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline +from scrapy.utils.asyncio import run_in_thread from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict +from scrapy.utils.defer import deferred_from_coro, ensure_awaitable from scrapy.utils.ftp import ftp_store_file +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str if TYPE_CHECKING: + from collections.abc import Awaitable from os import PathLike from twisted.python.failure import Failure @@ -159,7 +162,7 @@ class S3FilesStore: AWS_VERIFY = None POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler() - HEADERS = { + HEADERS: ClassVar[dict[str, str]] = { "Cache-Control": "max-age=172800", } @@ -183,26 +186,27 @@ class S3FilesStore: raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split("/", 1) + @staticmethod + def _onsuccess(boto_key: dict[str, Any]) -> StatInfo: + checksum = boto_key["ETag"].strip('"') + last_modified = boto_key["LastModified"] + modified_stamp = time.mktime(last_modified.timetuple()) + return {"checksum": checksum, "last_modified": modified_stamp} + def stat_file( self, path: str, info: MediaPipeline.SpiderInfo ) -> Deferred[StatInfo]: - def _onsuccess(boto_key: dict[str, Any]) -> StatInfo: - checksum = boto_key["ETag"].strip('"') - last_modified = boto_key["LastModified"] - modified_stamp = time.mktime(last_modified.timetuple()) - return {"checksum": checksum, "last_modified": modified_stamp} - return self._get_boto_key(path).addCallback(_onsuccess) + return self._get_boto_key(path).addCallback(self._onsuccess) def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]: key_name = f"{self.prefix}{path}" - return cast( - "Deferred[dict[str, Any]]", - deferToThread( + return deferred_from_coro( + run_in_thread( self.s3_client.head_object, # type: ignore[attr-defined] Bucket=self.bucket, Key=key_name, - ), + ) ) def persist_file( @@ -219,14 +223,16 @@ class S3FilesStore: extra = self._headers_to_botocore_kwargs(self.HEADERS) if headers: extra.update(self._headers_to_botocore_kwargs(headers)) - return deferToThread( - self.s3_client.put_object, # type: ignore[attr-defined] - Bucket=self.bucket, - Key=key_name, - Body=buf, - Metadata={k: str(v) for k, v in (meta or {}).items()}, - ACL=self.POLICY, - **extra, + return deferred_from_coro( + run_in_thread( + self.s3_client.put_object, # type: ignore[attr-defined] + Bucket=self.bucket, + Key=key_name, + Body=buf, + Metadata={k: str(v) for k, v in meta.items()} if meta else {}, + ACL=self.POLICY, + **extra, + ) ) def _headers_to_botocore_kwargs(self, headers: dict[str, Any]) -> dict[str, Any]: @@ -266,7 +272,9 @@ class S3FilesStore: try: kwarg = mapping[key] except KeyError: - raise TypeError(f'Header "{key}" is not supported by botocore') + raise TypeError( + f'Header "{key}" is not supported by botocore' + ) from None extra[kwarg] = value return extra @@ -302,21 +310,23 @@ class GCSFilesStore: {"bucket": bucket}, ) + @staticmethod + def _onsuccess(blob: Any) -> StatInfo: + if blob: + checksum = base64.b64decode(blob.md5_hash).hex() + last_modified = time.mktime(blob.updated.timetuple()) + return {"checksum": checksum, "last_modified": last_modified} + return {} + def stat_file( self, path: str, info: MediaPipeline.SpiderInfo ) -> Deferred[StatInfo]: - def _onsuccess(blob: Any) -> StatInfo: - if blob: - checksum = base64.b64decode(blob.md5_hash).hex() - last_modified = time.mktime(blob.updated.timetuple()) - return {"checksum": checksum, "last_modified": last_modified} - return {} blob_path = self._get_blob_path(path) - return cast( - "Deferred[StatInfo]", - deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess), + d: Deferred[Any] = deferred_from_coro( + run_in_thread(self.bucket.get_blob, blob_path) ) + return d.addCallback(self._onsuccess) def _get_content_type(self, headers: dict[str, str] | None) -> str: if headers and "Content-Type" in headers: @@ -337,12 +347,14 @@ class GCSFilesStore: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL - blob.metadata = {k: str(v) for k, v in (meta or {}).items()} - return deferToThread( - blob.upload_from_string, - data=buf.getvalue(), - content_type=self._get_content_type(headers), - predefined_acl=self.POLICY, + blob.metadata = {k: str(v) for k, v in meta.items()} if meta else {} + return deferred_from_coro( + run_in_thread( + blob.upload_from_string, + data=buf.getvalue(), + content_type=self._get_content_type(headers), + predefined_acl=self.POLICY, + ) ) @@ -375,37 +387,39 @@ class FTPFilesStore: headers: dict[str, str] | None = None, ) -> Deferred[Any]: path = f"{self.basedir}/{path}" - return deferToThread( - ftp_store_file, - path=path, - file=buf, - host=self.host, - port=self.port, - username=self.username, - password=self.password, - use_active_mode=self.USE_ACTIVE_MODE, + return deferred_from_coro( + run_in_thread( + ftp_store_file, + path=path, + file=buf, + host=self.host, + port=self.port, + username=self.username, + password=self.password, + use_active_mode=bool(self.USE_ACTIVE_MODE), + ) ) + def _stat_file(self, path: str) -> StatInfo: + try: + with FTP() as ftp: + ftp.connect(self.host, self.port) + ftp.login(self.username, self.password) + if self.USE_ACTIVE_MODE: + ftp.set_pasv(False) + file_path = f"{self.basedir}/{path}" + last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) + m = hashlib.md5() # noqa: S324 + ftp.retrbinary(f"RETR {file_path}", m.update) + return {"last_modified": last_modified, "checksum": m.hexdigest()} + # The file doesn't exist + except Exception: + return {} + def stat_file( self, path: str, info: MediaPipeline.SpiderInfo ) -> Deferred[StatInfo]: - def _stat_file(path: str) -> StatInfo: - try: - with FTP() as ftp: - ftp.connect(self.host, self.port) - ftp.login(self.username, self.password) - if self.USE_ACTIVE_MODE: - ftp.set_pasv(False) - file_path = f"{self.basedir}/{path}" - last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) - m = hashlib.md5() # noqa: S324 - ftp.retrbinary(f"RETR {file_path}", m.update) - return {"last_modified": last_modified, "checksum": m.hexdigest()} - # The file doesn't exist - except Exception: - return {} - - return cast("Deferred[StatInfo]", deferToThread(_stat_file, path)) + return deferred_from_coro(run_in_thread(self._stat_file, path)) class FilesPipeline(MediaPipeline): @@ -429,7 +443,7 @@ class FilesPipeline(MediaPipeline): MEDIA_NAME: str = "file" EXPIRES: int = 90 - STORE_SCHEMES: dict[str, type[FilesStoreProtocol]] = { + STORE_SCHEMES: ClassVar[dict[str, type[FilesStoreProtocol]]] = { "": FSFilesStore, "file": FSFilesStore, "s3": S3FilesStore, @@ -525,43 +539,51 @@ class FilesPipeline(MediaPipeline): store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) + def _onsuccess( + self, + result: StatInfo, + request: Request, + info: MediaPipeline.SpiderInfo, + path: str, + ) -> FileInfo | None: + if not result: + return None # returning None force download + + last_modified = result.get("last_modified", None) + if not last_modified: + return None # returning None force download + + age_seconds = time.time() - last_modified + age_days = age_seconds / 60 / 60 / 24 + if age_days > self.expires: + return None # returning None force download + + referer = referer_str(request) + logger.debug( + "File (uptodate): Downloaded %(medianame)s from %(request)s " + "referred in <%(referer)s>", + {"medianame": self.MEDIA_NAME, "request": request, "referer": referer}, + extra={"spider": info.spider}, + ) + self.inc_stats("uptodate") + + checksum = result.get("checksum", None) + return { + "url": request.url, + "path": path, + "checksum": checksum, + "status": "uptodate", + } + def media_to_download( self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None ) -> Deferred[FileInfo | None] | None: - def _onsuccess(result: StatInfo) -> FileInfo | None: - if not result: - return None # returning None force download - - last_modified = result.get("last_modified", None) - if not last_modified: - return None # returning None force download - - age_seconds = time.time() - last_modified - age_days = age_seconds / 60 / 60 / 24 - if age_days > self.expires: - return None # returning None force download - - referer = referer_str(request) - logger.debug( - "File (uptodate): Downloaded %(medianame)s from %(request)s " - "referred in <%(referer)s>", - {"medianame": self.MEDIA_NAME, "request": request, "referer": referer}, - extra={"spider": info.spider}, - ) - self.inc_stats("uptodate") - - checksum = result.get("checksum", None) - return { - "url": request.url, - "path": path, - "checksum": checksum, - "status": "uptodate", - } - path = self.file_path(request, info=info, item=item) # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload] - dfd2: Deferred[FileInfo | None] = dfd.addCallback(_onsuccess) + dfd2: Deferred[FileInfo | None] = dfd.addCallback( + functools.partial(self._onsuccess, request=request, info=info, path=path) + ) dfd2.addErrback(lambda _: None) dfd2.addErrback( lambda f: logger.error( @@ -591,7 +613,7 @@ class FilesPipeline(MediaPipeline): raise FileException - def media_downloaded( + async def media_downloaded( self, response: Response, request: Request, @@ -630,7 +652,9 @@ class FilesPipeline(MediaPipeline): try: path = self.file_path(request, response=response, info=info, item=item) - checksum = self.file_downloaded(response, request, info, item=item) + checksum: str = await ensure_awaitable( + self.file_downloaded(response, request, info, item=item) + ) except FileException as exc: logger.warning( "File (error): Error processing file from %(request)s " @@ -648,7 +672,7 @@ class FilesPipeline(MediaPipeline): exc_info=True, extra={"spider": info.spider}, ) - raise FileException(str(exc)) + raise FileException(str(exc)) from exc return { "url": request.url, @@ -662,6 +686,21 @@ class FilesPipeline(MediaPipeline): self.crawler.stats.inc_value("file_count") self.crawler.stats.inc_value(f"file_status_count/{status}") + async def _file_downloaded( + self, + response: Response, + request: Request, + info: MediaPipeline.SpiderInfo, + *, + item: Any = None, + ) -> str: + path = self.file_path(request, response=response, info=info, item=item) + buf = BytesIO(response.body) + checksum = _md5sum(buf) + buf.seek(0) + await ensure_awaitable(self.store.persist_file(path, buf, info)) + return checksum + # Overridable Interface def get_media_requests( self, item: Any, info: MediaPipeline.SpiderInfo @@ -680,13 +719,8 @@ class FilesPipeline(MediaPipeline): info: MediaPipeline.SpiderInfo, *, item: Any = None, - ) -> str: - path = self.file_path(request, response=response, info=info, item=item) - buf = BytesIO(response.body) - checksum = _md5sum(buf) - buf.seek(0) - self.store.persist_file(path, buf, info) - return checksum + ) -> str | Awaitable[str]: + return self._file_downloaded(response, request, info, item=item) def item_completed( self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo @@ -704,7 +738,15 @@ class FilesPipeline(MediaPipeline): item: Any = None, ) -> str: media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # noqa: S324 - media_ext = Path(request.url).suffix + + # clean it up and look at the path first + parsed_url = urlparse_cached(request) + media_ext = Path(parsed_url.path).suffix + + # if path has no extension look at the raw URL + if media_ext not in mimetypes.types_map: + media_ext = Path(request.url).suffix + # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise if media_ext not in mimetypes.types_map: diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index f5d1ad1b9..762b0fdf1 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,14 +11,21 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar, cast from itemadapter import ItemAdapter from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum +from scrapy.pipelines.files import ( + FileException, + FilesPipeline, + GCSFilesStore, + S3FilesStore, + _md5sum, +) +from scrapy.utils.defer import ensure_awaitable from scrapy.utils.python import to_bytes if TYPE_CHECKING: @@ -32,6 +39,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.pipelines.media import FileInfoOrError, MediaPipeline + from scrapy.settings import BaseSettings class ImageException(FileException): @@ -48,7 +56,7 @@ class ImagesPipeline(FilesPipeline): MIN_WIDTH: int = 0 MIN_HEIGHT: int = 0 EXPIRES: int = 90 - THUMBS: dict[str, tuple[int, int]] = {} + THUMBS: ClassVar[dict[str, tuple[int, int]]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" @@ -75,7 +83,7 @@ class ImagesPipeline(FilesPipeline): except ImportError: raise NotConfigured( "ImagesPipeline requires installing Pillow 8.3.2 or later" - ) + ) from None super().__init__(store_uri, crawler=crawler) @@ -115,7 +123,7 @@ class ImagesPipeline(FilesPipeline): store_uri = settings["IMAGES_STORE"] return cls(store_uri, crawler=crawler) - def file_downloaded( + async def file_downloaded( self, response: Response, request: Request, @@ -123,9 +131,23 @@ class ImagesPipeline(FilesPipeline): *, item: Any = None, ) -> str: - return self.image_downloaded(response, request, info, item=item) + return await self.image_downloaded(response, request, info, item=item) - def image_downloaded( + @classmethod + def _update_stores(cls, settings: BaseSettings) -> None: + super()._update_stores(settings) + + s3store: type[S3FilesStore] = cast( + "type[S3FilesStore]", cls.STORE_SCHEMES["s3"] + ) + s3store.POLICY = settings["IMAGES_STORE_S3_ACL"] + + gcs_store: type[GCSFilesStore] = cast( + "type[GCSFilesStore]", cls.STORE_SCHEMES["gs"] + ) + gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None + + async def image_downloaded( self, response: Response, request: Request, @@ -139,12 +161,14 @@ class ImagesPipeline(FilesPipeline): buf.seek(0) checksum = _md5sum(buf) width, height = image.size - self.store.persist_file( - path, - buf, - info, - meta={"width": width, "height": height}, - headers={"Content-Type": "image/jpeg"}, + await ensure_awaitable( + self.store.persist_file( + path, + buf, + info, + meta={"width": width, "height": height}, + headers={"Content-Type": "image/jpeg"}, + ) ) assert checksum is not None return checksum @@ -188,7 +212,7 @@ class ImagesPipeline(FilesPipeline): *, response_body: BytesIO, ) -> tuple[Image.Image, BytesIO]: - if image.format in ("PNG", "WEBP") and image.mode == "RGBA": + if image.format in {"PNG", "WEBP"} and image.mode == "RGBA": background = self._Image.new("RGBA", image.size, (255, 255, 255)) background.paste(image, image) image = background.convert("RGB") diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 7bc29f28f..5b5d2dcb2 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -8,13 +8,12 @@ from abc import ABC, abstractmethod from collections import defaultdict from typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypedDict, cast -from twisted import version as twisted_version from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure -from twisted.python.versions import Version from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.request import NO_CALLBACK, Request +from scrapy.utils._deps_compat import TWISTED_FAILURE_HAS_STACK from scrapy.utils.asyncio import call_later, is_asyncio_available from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.decorators import _warn_spider_arg @@ -30,6 +29,8 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import global_object_name if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + # typing.Self requires Python 3.11 from typing_extensions import Self @@ -152,7 +153,7 @@ class MediaPipeline(ABC): ) -> FileInfo: fp = self._fingerprinter.fingerprint(request) - eb = request.errback + eb: Callable[[Failure], FileInfo] | None = request.errback request.callback = NO_CALLBACK request.errback = None @@ -209,7 +210,9 @@ class MediaPipeline(ABC): self._modify_media_request(request) assert self.crawler.engine response = await self.crawler.engine.download_async(request) - return self.media_downloaded(response, request, info, item=item) + return await ensure_awaitable( + self.media_downloaded(response, request, info, item=item) + ) except Exception: failure = self.media_failed(Failure(), request, info) if isinstance(failure, Failure): @@ -227,9 +230,9 @@ class MediaPipeline(ABC): if isinstance(result, Failure): # minimize cached information for failure result.cleanFailure() - result.frames = [] - if twisted_version < Version("twisted", 24, 10, 0): - result.stack = [] # type: ignore[method-assign] + result.frames.clear() + if TWISTED_FAILURE_HAS_STACK: + result.stack.clear() # This code fixes a memory leak by avoiding to keep references to # the Request and Response objects on the Media Pipeline cache. # @@ -282,7 +285,7 @@ class MediaPipeline(ABC): info: SpiderInfo, *, item: Any = None, - ) -> FileInfo: + ) -> FileInfo | Awaitable[FileInfo]: """Handler for success downloads""" raise NotImplementedError diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index ed57091ba..0ad0b5d78 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -350,8 +350,37 @@ class DownloaderAwarePriorityQueue: self.crawler: Crawler = crawler self.pqueues: dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue - for slot, startprios in (slot_startprios or {}).items(): - self.pqueues[slot] = self.pqfactory(slot, startprios) + self._last_selected_slot: str | None = None + if slot_startprios: + for slot, startprios in slot_startprios.items(): + self.pqueues[slot] = self.pqfactory(slot, startprios) + + def _next_slot(self, stats: list[tuple[int, str]], *, update_state: bool) -> str: + last = self._last_selected_slot + min_active: int | None = None + best_slot: str | None = None + best_slot_after_last: str | None = None + for active, slot in stats: + if min_active is None or active < min_active: + min_active = active + best_slot = slot + best_slot_after_last = None + if last is not None and slot > last: + best_slot_after_last = slot + elif active == min_active: + if best_slot is None or slot < best_slot: + best_slot = slot + if ( + last is not None + and slot > last + and (best_slot_after_last is None or slot < best_slot_after_last) + ): + best_slot_after_last = slot + assert best_slot is not None + slot = best_slot_after_last if best_slot_after_last is not None else best_slot + if update_state: + self._last_selected_slot = slot + return slot def pqfactory( self, slot: str, startprios: Iterable[int] = () @@ -370,7 +399,7 @@ class DownloaderAwarePriorityQueue: if not stats: return None - slot = min(stats)[1] + slot = self._next_slot(stats, update_state=True) queue = self.pqueues[slot] request = queue.pop() if len(queue) == 0: @@ -394,7 +423,7 @@ class DownloaderAwarePriorityQueue: stats = self._downloader_interface.stats(self.pqueues) if not stats: return None - slot = min(stats)[1] + slot = self._next_slot(stats, update_state=False) queue = self.pqueues[slot] return queue.peek() diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f5f00ab0f..270a7fbf5 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -75,7 +75,7 @@ class HostResolution: def __init__(self, name: str): self.name: str = name - def cancel(self) -> None: + def cancel(self) -> None: # pragma: no cover raise NotImplementedError diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 3aaf17b53..29e9b6bb7 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -8,7 +8,7 @@ from __future__ import annotations from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar from scrapy.http import Response from scrapy.utils.misc import load_object @@ -19,7 +19,7 @@ if TYPE_CHECKING: class ResponseTypes: - CLASSES = { + CLASSES: ClassVar[dict[str, str]] = { "text/html": "scrapy.http.HtmlResponse", "application/atom+xml": "scrapy.http.XmlResponse", "application/rdf+xml": "scrapy.http.XmlResponse", @@ -40,18 +40,13 @@ class ResponseTypes: self.classes: dict[str, type[Response]] = {} self.mimetypes: MimeTypes = MimeTypes() mimedata = get_data("scrapy", "mime.types") - if not mimedata: - raise ValueError( - "The mime.types file is not found in the Scrapy installation" - ) + assert mimedata is not None self.mimetypes.readfp(StringIO(mimedata.decode("utf8"))) for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) def from_mimetype(self, mimetype: str) -> type[Response]: """Return the most appropriate Response class for the given mimetype""" - if mimetype is None: - return Response if mimetype in self.classes: return self.classes[mimetype] basetype = f"{mimetype.split('/', maxsplit=1)[0]}/*" diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 18b622546..0c64ea5a5 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import sys from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from urllib.robotparser import RobotFileParser from protego import Protego @@ -103,7 +103,7 @@ class RerpRobotParser(RobotParser): def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) - return self.rp.is_allowed(user_agent, url) + return cast("bool", self.rp.is_allowed(user_agent, url)) class ProtegoRobotParser(RobotParser): diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 598a746a9..be298e9b1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -16,10 +16,6 @@ from scrapy.utils.python import global_object_name logger = getLogger(__name__) -# The key types are restricted in BaseSettings._get_key() to ones supported by JSON, -# see https://github.com/scrapy/scrapy/issues/5383. -_SettingsKey: TypeAlias = bool | float | int | str | None - if TYPE_CHECKING: from types import ModuleType @@ -29,7 +25,7 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self - _SettingsInput: TypeAlias = SupportsItems[_SettingsKey, Any] | str | None + _SettingsInput: TypeAlias = SupportsItems[str, Any] | str | None SETTINGS_PRIORITIES: dict[str, int] = { @@ -80,7 +76,7 @@ class SettingsAttribute: return f"" -class BaseSettings(MutableMapping[_SettingsKey, Any]): +class BaseSettings(MutableMapping[str, Any]): """ Instances of this class behave like dictionaries, but store priorities along with their ``(key, value)`` pairs, and can be frozen (i.e. marked @@ -106,11 +102,11 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): def __init__(self, values: _SettingsInput = None, priority: int | str = "project"): self.frozen: bool = False - self.attributes: dict[_SettingsKey, SettingsAttribute] = {} + self.attributes: dict[str, SettingsAttribute] = {} if values: self.update(values, priority) - def __getitem__(self, opt_name: _SettingsKey) -> Any: + def __getitem__(self, opt_name: str) -> Any: if opt_name not in self: return None return self.attributes[opt_name].value @@ -118,7 +114,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): def __contains__(self, name: Any) -> bool: return name in self.attributes - def add_to_list(self, name: _SettingsKey, item: Any) -> None: + def add_to_list(self, name: str, item: Any) -> None: """Append *item* to the :class:`list` setting with the specified *name* if *item* is not already in that list. @@ -129,7 +125,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): if item not in value: self.set(name, [*value, item], self.getpriority(name) or 0) - def remove_from_list(self, name: _SettingsKey, item: Any) -> None: + def remove_from_list(self, name: str, item: Any) -> None: """Remove *item* from the :class:`list` setting with the specified *name*. @@ -143,7 +139,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).") self.set(name, [v for v in value if v != item], self.getpriority(name) or 0) - def get(self, name: _SettingsKey, default: Any = None) -> Any: + def get(self, name: str, default: Any = None) -> Any: # pylint: disable=arguments-renamed """ Get a setting value without affecting its original type. @@ -162,9 +158,17 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): stacklevel=2, ) + if name == "DNS_RESOLVER": + warnings.warn( + "The DNS_RESOLVER setting is deprecated, please use " + "TWISTED_DNS_RESOLVER instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return self[name] if self[name] is not None else default - def getbool(self, name: _SettingsKey, default: bool = False) -> bool: + def getbool(self, name: str, default: bool = False) -> bool: """ Get a setting value as a boolean. @@ -184,17 +188,17 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): try: return bool(int(got)) except ValueError: - if got in ("True", "true"): + if got in {"True", "true"}: return True - if got in ("False", "false"): + if got in {"False", "false"}: return False raise ValueError( "Supported values for boolean settings " "are 0/1, True/False, '0'/'1', " "'True'/'False' and 'true'/'false'" - ) + ) from None - def getint(self, name: _SettingsKey, default: int = 0) -> int: + def getint(self, name: str, default: int = 0) -> int: """ Get a setting value as an int. @@ -206,7 +210,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): """ return int(self.get(name, default)) - def getfloat(self, name: _SettingsKey, default: float = 0.0) -> float: + def getfloat(self, name: str, default: float = 0.0) -> float: """ Get a setting value as a float. @@ -218,9 +222,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): """ return float(self.get(name, default)) - def getlist( - self, name: _SettingsKey, default: list[Any] | None = None - ) -> list[Any]: + def getlist(self, name: str, default: list[Any] | None = None) -> list[Any]: """ Get a setting value as a list. If the setting original type is a list, a copy of it will be returned. If it's a string it will be split by @@ -243,7 +245,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): return list(value) def getdict( - self, name: _SettingsKey, default: dict[Any, Any] | None = None + self, name: str, default: dict[Any, Any] | None = None ) -> dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a @@ -267,7 +269,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): def getdictorlist( self, - name: _SettingsKey, + name: str, default: dict[Any, Any] | list[Any] | tuple[Any] | None = None, ) -> dict[Any, Any] | list[Any]: """Get a setting value as either a :class:`dict` or a :class:`list`. @@ -314,13 +316,35 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): ) return copy.deepcopy(value) - def getwithbase(self, name: _SettingsKey) -> BaseSettings: - """Get a composition of a dictionary-like setting and its `_BASE` + def getwithbase(self, name: str) -> BaseSettings: + """Get a composition of a dictionary-like setting and its ``_BASE`` counterpart. + Use + :meth:`~scrapy.settings.BaseSettings.get_component_priority_dict_with_base` + instead if the setting is a :ref:`component priority dictionary + `. + :param name: name of the dictionary-like setting :type name: str """ + if not isinstance(name, str): + raise ValueError(f"Base setting key must be a string, got {name}") + compbs = BaseSettings() + compbs.update(self[name + "_BASE"]) + compbs.update(self[name]) + return compbs + + def get_component_priority_dict_with_base(self, name: str) -> BaseSettings: + """Get a composition of a component priority dictionary setting and + its ``_BASE`` counterpart. + + Keys are resolved to their import path for deduplication and then + restored to their latest input representation. + + :param name: name of the component priority dictionary setting + :type name: str + """ if not isinstance(name, str): raise ValueError(f"Base setting key must be a string, got {name}") @@ -337,10 +361,10 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): f"be kept." ) - def normalize_key(key: Any) -> str: + def normalize_key(key: Any) -> Any: try: loaded_key = load_object(key) - except (AttributeError, TypeError, ValueError): + except (NameError, TypeError, ValueError): loaded_key = key else: import_path = global_object_name(loaded_key) @@ -359,7 +383,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): {restore_key(k): v for k, v in result.items() if v is not None} ) - def getpriority(self, name: _SettingsKey) -> int | None: + def getpriority(self, name: str) -> int | None: """ Return the current numerical priority value of a setting, or ``None`` if the given ``name`` does not exist. @@ -384,7 +408,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): def replace_in_component_priority_dict( self, - name: _SettingsKey, + name: str, old_cls: type, new_cls: type, priority: int | None = None, @@ -423,12 +447,10 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): ) self.set(name, component_priority_dict, priority=self.getpriority(name) or 0) - def __setitem__(self, name: _SettingsKey, value: Any) -> None: + def __setitem__(self, name: str, value: Any) -> None: self.set(name, value) - def set( - self, name: _SettingsKey, value: Any, priority: int | str = "project" - ) -> None: + def set(self, name: str, value: Any, priority: int | str = "project") -> None: """ Store a key/value attribute with a given priority. @@ -457,7 +479,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): self.attributes[name].set(value, priority) def set_in_component_priority_dict( - self, name: _SettingsKey, cls: type, priority: int | None + self, name: str, cls: type, priority: int | None ) -> None: """Set the *cls* component in the *name* :ref:`component priority dictionary ` setting with *priority*. @@ -480,9 +502,9 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): component_priority_dict[cls] = priority self.set(name, component_priority_dict, self.getpriority(name) or 0) - def setdefault( + def setdefault( # pylint: disable=arguments-renamed self, - name: _SettingsKey, + name: str, default: Any = None, priority: int | str = "project", ) -> Any: @@ -493,7 +515,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): return self.attributes[name].value def setdefault_in_component_priority_dict( - self, name: _SettingsKey, cls: type, priority: int | None + self, name: str, cls: type, priority: int | None ) -> None: """Set the *cls* component in the *name* :ref:`component priority dictionary ` setting with *priority* @@ -562,7 +584,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast("dict[_SettingsKey, Any]", json.loads(values)) + values = cast("dict[str, Any]", json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): @@ -571,7 +593,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): for name, value in values.items(): self.set(name, value, priority) - def delete(self, name: _SettingsKey, priority: int | str = "project") -> None: + def delete(self, name: str, priority: int | str = "project") -> None: if name not in self: raise KeyError(name) self._assert_mutability() @@ -579,7 +601,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): if priority >= cast("int", self.getpriority(name)): del self.attributes[name] - def __delitem__(self, name: _SettingsKey) -> None: + def __delitem__(self, name: str) -> None: self._assert_mutability() del self.attributes[name] @@ -619,26 +641,19 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): copy.freeze() return copy - def __iter__(self) -> Iterator[_SettingsKey]: + def __iter__(self) -> Iterator[str]: return iter(self.attributes) def __len__(self) -> int: return len(self.attributes) - def _to_dict(self) -> dict[_SettingsKey, Any]: + def _to_dict(self) -> dict[str, Any]: return { - self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) + str(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() } - def _get_key(self, key_value: Any) -> _SettingsKey: - return ( - key_value - if isinstance(key_value, (bool, float, int, str, type(None))) - else str(key_value) - ) - - def copy_to_dict(self) -> dict[_SettingsKey, Any]: + def copy_to_dict(self) -> dict[str, Any]: """ Make a copy of current settings and convert to a dict. @@ -661,14 +676,14 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): else: p.text(pformat(self.copy_to_dict())) - def pop(self, name: _SettingsKey, default: Any = __default) -> Any: + def pop(self, name: str, default: Any = __default) -> Any: # pylint: disable=arguments-renamed try: value = self.attributes[name].value except KeyError: if default is self.__default: raise return default - self.__delitem__(name) + del self[name] return value @@ -705,7 +720,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]: def overridden_settings( - settings: Mapping[_SettingsKey, Any], + settings: Mapping[str, Any], ) -> Iterable[tuple[str, Any]]: """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 385d5f933..049921ba7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -16,22 +16,30 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module from pathlib import Path +from typing import Any __all__ = [ "ADDONS", - "AJAXCRAWL_ENABLED", - "AJAXCRAWL_MAXSIZE", "ASYNCIO_EVENT_LOOP", "AUTOTHROTTLE_DEBUG", "AUTOTHROTTLE_ENABLED", "AUTOTHROTTLE_MAX_DELAY", "AUTOTHROTTLE_START_DELAY", "AUTOTHROTTLE_TARGET_CONCURRENCY", + "AWS_ACCESS_KEY_ID", + "AWS_ENDPOINT_URL", + "AWS_REGION_NAME", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_USE_SSL", + "AWS_VERIFY", "BOT_NAME", "CLOSESPIDER_ERRORCOUNT", "CLOSESPIDER_ITEMCOUNT", "CLOSESPIDER_PAGECOUNT", + "CLOSESPIDER_PAGECOUNT_NO_ITEM", "CLOSESPIDER_TIMEOUT", + "CLOSESPIDER_TIMEOUT_NO_ITEM", "COMMANDS_MODULE", "COMPRESSION_ENABLED", "CONCURRENT_ITEMS", @@ -48,25 +56,29 @@ __all__ = [ "DEPTH_STATS_VERBOSE", "DNSCACHE_ENABLED", "DNSCACHE_SIZE", - "DNS_RESOLVER", "DNS_TIMEOUT", "DOWNLOADER", "DOWNLOADER_CLIENTCONTEXTFACTORY", "DOWNLOADER_CLIENT_TLS_CIPHERS", "DOWNLOADER_CLIENT_TLS_METHOD", "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING", - "DOWNLOADER_HTTPCLIENTFACTORY", "DOWNLOADER_MIDDLEWARES", "DOWNLOADER_MIDDLEWARES_BASE", "DOWNLOADER_STATS", + "DOWNLOAD_BIND_ADDRESS", "DOWNLOAD_DELAY", "DOWNLOAD_FAIL_ON_DATALOSS", "DOWNLOAD_HANDLERS", "DOWNLOAD_HANDLERS_BASE", "DOWNLOAD_MAXSIZE", + "DOWNLOAD_SLOTS", "DOWNLOAD_TIMEOUT", + "DOWNLOAD_TLS_MAX_VERSION", + "DOWNLOAD_TLS_MIN_VERSION", + "DOWNLOAD_VERIFY_CERTIFICATES", "DOWNLOAD_WARNSIZE", "DUPEFILTER_CLASS", + "DUPEFILTER_DEBUG", "EDITOR", "EXTENSIONS", "EXTENSIONS_BASE", @@ -85,7 +97,9 @@ __all__ = [ "FEED_STORAGE_S3_ACL", "FEED_STORE_EMPTY", "FEED_TEMPDIR", + "FEED_URI", "FEED_URI_PARAMS", + "FILES_STORE", "FILES_STORE_GCS_ACL", "FILES_STORE_S3_ACL", "FORCE_CRAWLER_PROCESS", @@ -93,6 +107,9 @@ __all__ = [ "FTP_PASSWORD", "FTP_USER", "GCS_PROJECT_ID", + "HTTPAUTH_DOMAIN", + "HTTPAUTH_PASS", + "HTTPAUTH_USER", "HTTPCACHE_ALWAYS_STORE", "HTTPCACHE_DBM_MODULE", "HTTPCACHE_DIR", @@ -105,8 +122,12 @@ __all__ = [ "HTTPCACHE_IGNORE_SCHEMES", "HTTPCACHE_POLICY", "HTTPCACHE_STORAGE", + "HTTPERROR_ALLOWED_CODES", + "HTTPERROR_ALLOW_ALL", "HTTPPROXY_AUTH_ENCODING", "HTTPPROXY_ENABLED", + "HTTPX_HTTP2_ENABLED", + "IMAGES_STORE", "IMAGES_STORE_GCS_ACL", "IMAGES_STORE_S3_ACL", "ITEM_PIPELINES", @@ -129,6 +150,8 @@ __all__ = [ "MAIL_HOST", "MAIL_PASS", "MAIL_PORT", + "MAIL_SSL", + "MAIL_TLS", "MAIL_USER", "MEMDEBUG_ENABLED", "MEMDEBUG_NOTIFY", @@ -150,10 +173,12 @@ __all__ = [ "REDIRECT_MAX_TIMES", "REDIRECT_PRIORITY_ADJUST", "REFERER_ENABLED", + "REFERRER_POLICIES", "REFERRER_POLICY", "REQUEST_FINGERPRINTER_CLASS", "RETRY_ENABLED", "RETRY_EXCEPTIONS", + "RETRY_GIVE_UP_LOG_LEVEL", "RETRY_HTTP_CODES", "RETRY_PRIORITY_ADJUST", "RETRY_TIMES", @@ -184,8 +209,9 @@ __all__ = [ "TELNETCONSOLE_PORT", "TELNETCONSOLE_USERNAME", "TEMPLATES_DIR", - "TWISTED_ENABLED", + "TWISTED_DNS_RESOLVER", "TWISTED_REACTOR", + "TWISTED_REACTOR_ENABLED", "URLLENGTH_LIMIT", "USER_AGENT", "WARN_ON_GENERATOR_RETURN_VALUE", @@ -193,9 +219,6 @@ __all__ = [ ADDONS = {} -AJAXCRAWL_ENABLED = False -AJAXCRAWL_MAXSIZE = 32768 - ASYNCIO_EVENT_LOOP = None AUTOTHROTTLE_ENABLED = False @@ -204,12 +227,22 @@ AUTOTHROTTLE_MAX_DELAY = 60.0 AUTOTHROTTLE_START_DELAY = 5.0 AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +AWS_ACCESS_KEY_ID = None +AWS_SECRET_ACCESS_KEY = None +AWS_ENDPOINT_URL = None +AWS_REGION_NAME = None +AWS_SESSION_TOKEN = None +AWS_USE_SSL = None +AWS_VERIFY = None + BOT_NAME = "scrapybot" CLOSESPIDER_ERRORCOUNT = 0 CLOSESPIDER_ITEMCOUNT = 0 CLOSESPIDER_PAGECOUNT = 0 CLOSESPIDER_TIMEOUT = 0 +CLOSESPIDER_PAGECOUNT_NO_ITEM = 0 +CLOSESPIDER_TIMEOUT_NO_ITEM = 0 COMMANDS_MODULE = "" @@ -243,6 +276,8 @@ DNSCACHE_SIZE = 10000 DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver" DNS_TIMEOUT = 60 +DOWNLOAD_BIND_ADDRESS = None + DOWNLOAD_DELAY = 0 DOWNLOAD_FAIL_ON_DATALOSS = True @@ -260,22 +295,22 @@ DOWNLOAD_HANDLERS_BASE = { DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m +DOWNLOAD_SLOTS = {} + DOWNLOAD_TIMEOUT = 180 # 3mins +DOWNLOAD_TLS_MAX_VERSION = None +DOWNLOAD_TLS_MIN_VERSION = None + +DOWNLOAD_VERIFY_CERTIFICATES = False + DOWNLOADER = "scrapy.core.downloader.Downloader" -DOWNLOADER_CLIENTCONTEXTFACTORY = ( - "scrapy.core.downloader.contextfactory.ScrapyClientContextFactory" -) +DOWNLOADER_CLIENTCONTEXTFACTORY = "SENTINEL" DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT" -# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation: DOWNLOADER_CLIENT_TLS_METHOD = "TLS" DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False -DOWNLOADER_HTTPCLIENTFACTORY = ( - "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory" -) - DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side @@ -286,7 +321,6 @@ DOWNLOADER_MIDDLEWARES_BASE = { "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400, "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500, "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550, - "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560, "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, @@ -300,6 +334,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { DOWNLOADER_STATS = True DUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter" +DUPEFILTER_DEBUG = False EDITOR = "vi" if sys.platform == "win32": @@ -350,8 +385,10 @@ FEED_STORAGE_FTP_ACTIVE = False FEED_STORAGE_GCS_ACL = "" FEED_STORAGE_S3_ACL = "" FEED_TEMPDIR = None +FEED_URI = None FEED_URI_PARAMS = None # a function to extend uri arguments +FILES_STORE = None FILES_STORE_GCS_ACL = "" FILES_STORE_S3_ACL = "private" @@ -363,6 +400,10 @@ FTP_PASSWORD = "guest" # noqa: S105 GCS_PROJECT_ID = None +HTTPAUTH_USER = "" +HTTPAUTH_PASS = "" +HTTPAUTH_DOMAIN = None + HTTPCACHE_ENABLED = False HTTPCACHE_ALWAYS_STORE = False HTTPCACHE_DBM_MODULE = "dbm" @@ -376,9 +417,15 @@ HTTPCACHE_IGNORE_SCHEMES = ["file"] HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" +HTTPERROR_ALLOW_ALL = False +HTTPERROR_ALLOWED_CODES = [] + HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = "latin-1" +HTTPX_HTTP2_ENABLED = False + +IMAGES_STORE = None IMAGES_STORE_GCS_ACL = "" IMAGES_STORE_S3_ACL = "private" @@ -419,6 +466,8 @@ MAIL_HOST = "localhost" MAIL_PORT = 25 MAIL_USER = None MAIL_PASS = None +MAIL_SSL = False +MAIL_TLS = False MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown @@ -449,6 +498,7 @@ REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" +REFERRER_POLICIES = {} REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" @@ -467,6 +517,7 @@ RETRY_EXCEPTIONS = [ OSError, "scrapy.core.downloader.handlers.http11.TunnelError", ] +RETRY_GIVE_UP_LOG_LEVEL = "ERROR" RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 RETRY_TIMES = 2 # initial response + 2 retries = 3 requests @@ -523,7 +574,9 @@ TELNETCONSOLE_PASSWORD = None TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) -TWISTED_ENABLED = True +TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver" + +TWISTED_REACTOR_ENABLED = True TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" URLLENGTH_LIMIT = 2083 @@ -533,7 +586,7 @@ USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org WARN_ON_GENERATOR_RETURN_VALUE = True -def __getattr__(name: str): +def __getattr__(name: str) -> Any: if name == "CONCURRENT_REQUESTS_PER_IP": import warnings # noqa: PLC0415 @@ -546,4 +599,4 @@ def __getattr__(name: str): ) return 0 - raise AttributeError + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/shell.py b/scrapy/shell.py index 6f33f3f14..44dcd880e 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -6,33 +6,90 @@ See documentation in docs/topics/shell.rst from __future__ import annotations +import asyncio import contextlib import os import signal +import warnings from typing import TYPE_CHECKING, Any from itemadapter import is_item -from twisted.internet import defer, threads +from twisted.internet import threads +from twisted.internet.defer import Deferred from twisted.python import threadable from w3lib.url import any_to_uri import scrapy from scrapy.crawler import Crawler -from scrapy.exceptions import IgnoreRequest +from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude -from scrapy.utils.defer import _schedule_coro, deferred_f_from_coro_f +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.misc import load_object -from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser if TYPE_CHECKING: from collections.abc import Callable +# Hopefully temporary architecture notes +# +# The Shell class is always instantiated in the "main" thread. There are two +# official ways to use it: +# 1. scrapy.commands.shell, which makes a secondary thread and calls +# CrawlerProcess.start() in it, which runs a reactor there. +# 2. scrapy.shell.inspect_response(), which just creates Shell() in the current +# thread. +# +# Shell._inthread is True when this class is run in a thread separate from the +# reactor, e.g. the 1st way (in other words, the reactor is in a secondary +# thread). +# Shell._inthread is False when this class is run in the same thread as the +# reactor, e.g. the 2nd way. +# The only thing that differs is availability of fetch() (it needs the +# reactor to be in a separate thread: the shell sends the request to +# the reactor and waits for the result synchronously). +# +# Thus the only thing Shell needs an event loop for is fetch(). More machinery +# is used for it to work. In chronological order: +# 1. scrapy.commands.shell.Command.run() creates a crawler and an engine, then +# calls +# _schedule_coro(crawler.engine.start_async(_start_request_processing=False)), +# which initializes the engine but doesn't start processing of requests. +# 2. scrapy.commands.shell.Command.run() calls crawler_process.start() in a +# thread which starts a reactor in that thread. +# 3. When fetch() is called, it prepares a request and calls Shell._schedule() +# in the reactor thread (via threads.blockingCallFromThread()). +# 4. Shell._schedule() calls Shell._open_spider() (on the first call). +# 5. Shell._open_spider() calls +# engine.open_spider_async(close_if_idle=False). +# 6. Shell._schedule() calls engine.crawl(request), scheduling the request. +# 7. Shell._schedule() via _request_deferred() waits until the request callback +# is called. When it's called, the response becomes available. +# +# In the reactorless mode this is slightly different, the engine initialization +# happens in the event loop thread as many things need either a reactor or a +# running event loop. +# +# Side note: it should be possible to remove _request_deferred() by using +# engine.download_async() instead of engine.schedule(), losing the usual stuff +# like spider middlewares (none of which should be important). +# +# Other architecture problems: +# * scrapy.cmdline.execute() creates an AsyncCrawlerProcess instance which +# immediately installs a reactor (which is maybe not thread-specific?) or an +# event loop (which *is* thread-specific, so the main thread will always have +# a (not running) loop installed. +# * scrapy.commands.shell.Command.run() calls _schedule_coro() in the main +# thread, and various engine init code also calls similar things, +# conceptually this shouldn't work (and doesn't in the reactorless mode, so +# there the initialization is moved to the event loop thread). +# * The engine has several code paths specifically for the shell, and the shell +# uses several private members of the engine and of AsyncCrawlerProcess. + class Shell: relevant_classes: tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) @@ -42,17 +99,47 @@ class Shell: crawler: Crawler, update_vars: Callable[[dict[str, Any]], None] | None = None, code: str | None = None, + *, + loop: asyncio.AbstractEventLoop | None = None, ): + self._use_reactor = crawler.settings.getbool("TWISTED_REACTOR_ENABLED") + if not self._use_reactor and not loop: # pragma: no cover + raise RuntimeError( + "Shell needs the crawler loop reference when TWISTED_REACTOR_ENABLED=False." + ) + self._loop = loop self.crawler: Crawler = crawler self.update_vars: Callable[[dict[str, Any]], None] = update_vars or ( lambda x: None ) self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) self.spider: Spider | None = None - self.inthread: bool = not threadable.isInIOThread() + if self._use_reactor: + self._inthread: bool = not threadable.isInIOThread() + else: + try: + # in case there is also a running loop in the main thread + current_loop = asyncio.get_running_loop() + self._inthread = current_loop is not self._loop + except RuntimeError: + self._inthread = True self.code: str | None = code self.vars: dict[str, Any] = {} + @property + def inthread(self) -> bool: # pragma: no cover + warnings.warn( + "Shell.inthread is deprecated, use Shell.fetch_available instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return self._inthread + + @property + def fetch_available(self) -> bool: + """Whether fetch() can be used.""" + return self._inthread + def start( self, url: str | None = None, @@ -98,35 +185,26 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule(self, request: Request, spider: Spider | None) -> defer.Deferred[Any]: - if is_asyncio_reactor_installed(): - # set the asyncio event loop for the current thread - event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] - set_asyncio_event_loop(event_loop_path) + async def _schedule(self, request: Request, spider: Spider | None) -> Response: + """Send the request to the engine, wait for the result. - def crawl_request(_: None) -> None: - assert self.crawler.engine is not None - self.crawler.engine.crawl(request) - - d2 = self._open_spider(request, spider) - d2.addCallback(crawl_request) - - d = _request_deferred(request) - d.addCallback(lambda x: (x, spider)) - return d - - @deferred_f_from_coro_f - async def _open_spider(self, request: Request, spider: Spider | None) -> None: - if self.spider: - return + Runs in the reactor thread. + """ + if not self.spider: + await self._open_spider(spider) + assert self.crawler.engine is not None + # send the request to the engine + self.crawler.engine.crawl(request) + # this will fire when the request callback runs (via the callback hijacking in _request_deferred()) + return await maybe_deferred_to_future(_request_deferred(request)) + async def _open_spider(self, spider: Spider | None) -> None: if spider is None: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider assert self.crawler.engine await self.crawler.engine.open_spider_async(close_if_idle=False) - _schedule_coro(self.crawler.engine._start_request_processing()) self.spider = spider def fetch( @@ -136,8 +214,6 @@ class Shell: redirect: bool = True, **kwargs: Any, ) -> None: - from twisted.internet import reactor - if isinstance(request_or_url, Request): request = request_or_url else: @@ -149,12 +225,25 @@ class Shell: ) else: request.meta["handle_httpstatus_all"] = True - response = None - with contextlib.suppress(IgnoreRequest): - response, spider = threads.blockingCallFromThread( - reactor, self._schedule, request, spider - ) - self.populate_vars(response, request, spider) + response: Response | None = None + if self._use_reactor: + from twisted.internet import reactor + + with contextlib.suppress(IgnoreRequest): + response = threads.blockingCallFromThread( + reactor, + deferred_f_from_coro_f(self._schedule), # type: ignore[arg-type] + request, + spider, + ) + else: + assert self._loop + with contextlib.suppress(IgnoreRequest): + future = asyncio.run_coroutine_threadsafe( + self._schedule(request, spider), self._loop + ) + response = future.result() + self.populate_vars(response, request, self.spider) def populate_vars( self, @@ -169,7 +258,7 @@ class Shell: self.vars["spider"] = spider self.vars["request"] = request self.vars["response"] = response - if self.inthread: + if self.fetch_available: self.vars["fetch"] = self.fetch self.vars["view"] = open_in_browser self.vars["shelp"] = self.print_help @@ -190,7 +279,7 @@ class Shell: if self._is_relevant(v): b.append(f" {k:<10} {v}") b.append("Useful shortcuts:") - if self.inthread: + if self.fetch_available: b.append( " fetch(url[, redirect=True]) " "Fetch URL and update local objects (by default, redirects are followed)" @@ -213,11 +302,15 @@ def inspect_response(response: Response, spider: Spider) -> None: # Shell.start removes the SIGINT handler, so save it and re-add it after # the shell has closed sigint_handler = signal.getsignal(signal.SIGINT) - Shell(spider.crawler).start(response=response, spider=spider) + if not spider.crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): + loop = asyncio.get_running_loop() + else: + loop = None + Shell(spider.crawler, loop=loop).start(response=response, spider=spider) signal.signal(signal.SIGINT, sigint_handler) -def _request_deferred(request: Request) -> defer.Deferred[Any]: +def _request_deferred(request: Request) -> Deferred[Any]: """Wrap a request inside a Deferred. This function is harmful, do not use it until you know what you are doing. @@ -236,7 +329,7 @@ def _request_deferred(request: Request) -> defer.Deferred[Any]: request.errback = request_errback return result - d: defer.Deferred[Any] = defer.Deferred() + d: Deferred[Any] = Deferred() d.addBoth(_restore_callbacks) if request.callback: d.addCallback(request.callback) diff --git a/scrapy/signals.py b/scrapy/signals.py index bdeec1ba0..972f4fd60 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -12,6 +12,7 @@ spider_opened = object() spider_idle = object() spider_closed = object() spider_error = object() +memusage_warning_reached = object() request_scheduled = object() request_dropped = object() request_reached_downloader = object() diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 8eac188c8..8c980fd46 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -9,7 +9,7 @@ from zope.interface import implementer from zope.interface.verify import verifyClass from scrapy.interfaces import ISpiderLoader -from scrapy.utils.misc import load_object, walk_modules +from scrapy.utils.misc import load_object, walk_modules_iter from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: @@ -77,6 +77,7 @@ class SpiderLoader: warnings.warn( "There are several spiders with the same name:\n\n" f"{dupes_string}\n\n This can cause unexpected behavior.", + stacklevel=2, category=UserWarning, ) @@ -88,7 +89,7 @@ class SpiderLoader: def _load_all_spiders(self) -> None: for name in self.spider_modules: try: - for module in walk_modules(name): + for module in walk_modules_iter(name): self._load_spiders(module) except (ImportError, SyntaxError): if self.warn_only: @@ -96,6 +97,7 @@ class SpiderLoader: f"\n{traceback.format_exc()}Could not load spiders " f"from module '{name}'. " "See above traceback for details.", + stacklevel=2, category=RuntimeWarning, ) else: @@ -114,7 +116,7 @@ class SpiderLoader: try: return self._spiders[spider_name] except KeyError: - raise KeyError(f"Spider not found: {spider_name}") + raise KeyError(f"Spider not found: {spider_name}") from None def find_by_request(self, request: Request) -> list[str]: """ diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index 889fc6df1..e09f2d10e 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -41,13 +41,6 @@ class BaseSpiderMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def process_start_requests( - self, start: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - for o in start: - if (o := self._get_processed(o, None)) is not None: - yield o - async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: async for o in start: if (o := self._get_processed(o, None)) is not None: diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index c160d1adb..94b6dfbb5 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -88,5 +88,5 @@ class HttpErrorMiddleware: {"response": response}, extra={"spider": self.crawler.spider}, ) - return [] + return () return None diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index ac340324a..2c9452174 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -9,24 +9,25 @@ import warnings from abc import ABC, abstractmethod from typing import TYPE_CHECKING, cast from urllib.parse import urlparse +from warnings import warn -from w3lib.url import safe_url_string - -from scrapy import Spider, signals -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.misc import load_object -from scrapy.utils.python import to_unicode +from scrapy.utils.python import _looks_like_import_path, to_unicode from scrapy.utils.url import strip_url if TYPE_CHECKING: # typing.Self requires Python 3.11 - from typing_extensions import Self + from typing_extensions import Self, TypedDict, Unpack from scrapy.crawler import Crawler from scrapy.settings import BaseSettings + class _PolicyKwargs(TypedDict, total=False): + resp_or_url: Response | str + LOCAL_SCHEMES: tuple[str, ...] = ( "about", @@ -102,7 +103,7 @@ class ReferrerPolicy(ABC): return self.tls_protected(url) def tls_protected(self, url: str) -> bool: - return urlparse(url).scheme in ("https", "ftps") + return urlparse(url).scheme in {"https", "ftps"} class NoReferrerPolicy(ReferrerPolicy): @@ -287,92 +288,145 @@ class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): name: str = POLICY_SCRAPY_DEFAULT -_policy_classes: dict[str, type[ReferrerPolicy]] = { - p.name: p - for p in ( - NoReferrerPolicy, - NoReferrerWhenDowngradePolicy, - SameOriginPolicy, - OriginPolicy, - StrictOriginPolicy, - OriginWhenCrossOriginPolicy, - StrictOriginWhenCrossOriginPolicy, - UnsafeUrlPolicy, - DefaultReferrerPolicy, - ) -} - -# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string -_policy_classes[""] = NoReferrerWhenDowngradePolicy - - -def _load_policy_class( - policy: str, warning_only: bool = False -) -> type[ReferrerPolicy] | None: - """ - Expect a string for the path to the policy class, - otherwise try to interpret the string as a standard value - from https://www.w3.org/TR/referrer-policy/#referrer-policies - """ - try: - return cast("type[ReferrerPolicy]", load_object(policy)) - except ValueError: - tokens = [token.strip() for token in policy.lower().split(",")] - # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header - for token in tokens[::-1]: - if token in _policy_classes: - return _policy_classes[token] - - msg = f"Could not load referrer policy {policy!r}" - if not warning_only: - raise RuntimeError(msg) - warnings.warn(msg, RuntimeWarning) - return None - - class RefererMiddleware(BaseSpiderMiddleware): def __init__(self, settings: BaseSettings | None = None): # pylint: disable=super-init-not-called self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy - if settings is not None: - settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) - assert settings_policy - self.default_policy = settings_policy + self.policies: dict[str, type[ReferrerPolicy]] = { + p.name: p + for p in ( + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + SameOriginPolicy, + OriginPolicy, + StrictOriginPolicy, + OriginWhenCrossOriginPolicy, + StrictOriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, + DefaultReferrerPolicy, + ) + } + # Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string + self.policies[""] = NoReferrerWhenDowngradePolicy + if settings is None: + return + setting_policies = settings.getdict("REFERRER_POLICIES") + for policy_name, policy_class_import_path in setting_policies.items(): + if policy_class_import_path is None: + del self.policies[policy_name] + else: + self.policies[policy_name] = load_object(policy_class_import_path) + settings_policy = self._load_policy_class( + settings.get("REFERRER_POLICY"), allow_import_path=True + ) + assert settings_policy + self.default_policy = settings_policy @classmethod def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("REFERER_ENABLED"): raise NotConfigured - mw = cls(crawler.settings) + return cls(crawler.settings) - # Note: this hook is a bit of a hack to intercept redirections - crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) - - return mw - - def policy(self, resp_or_url: Response | str, request: Request) -> ReferrerPolicy: - """ - Determine Referrer-Policy to use from a parent Response (or URL), - and a Request to be sent. + def policy( + self, + response: Response | str | None = None, + request: Request | None = None, + **kwargs: Unpack[_PolicyKwargs], + ) -> ReferrerPolicy: + """Return the referrer policy to use for *request* based on *request* + meta, *response* and settings. - if a valid policy is set in Request meta, it is used. - - if the policy is set in meta but is wrong (e.g. a typo error), - the policy from settings is used - - if the policy is not set in Request meta, - but there is a Referrer-policy header in the parent response, - it is used if valid + - if the policy is set in meta but is wrong (e.g. a typo error), the + policy from settings is used + - if the policy is not set in Request meta, but there is a + Referrer-Policy header in the parent response, it is used if valid - otherwise, the policy from settings is used. """ + if "resp_or_url" in kwargs: + if response is not None: + raise TypeError("Cannot pass both 'response' and 'resp_or_url'") + response = kwargs.pop("resp_or_url") + warn( + "Passing 'resp_or_url' is deprecated, use 'response' instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + if response is None: + raise TypeError("Missing required argument: 'response'") + if request is None: + raise TypeError("Missing required argument: 'request'") + if isinstance(response, str): + warn( + "Passing a response URL to RefererMiddleware.policy() instead " + "of a Response object is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + allow_import_path = True policy_name = request.meta.get("referrer_policy") - if policy_name is None and isinstance(resp_or_url, Response): - policy_header = resp_or_url.headers.get("Referrer-Policy") + if policy_name is None and isinstance(response, Response): + policy_header = response.headers.get("Referrer-Policy") if policy_header is not None: policy_name = to_unicode(policy_header.decode("latin1")) + allow_import_path = False if policy_name is None: return self.default_policy() - - cls = _load_policy_class(policy_name, warning_only=True) + cls = self._load_policy_class( + policy_name, warning_only=True, allow_import_path=allow_import_path + ) return cls() if cls else self.default_policy() + def _load_policy_class( + self, + policy: str, + warning_only: bool = False, + *, + allow_import_path: bool = False, + ) -> type[ReferrerPolicy] | None: + """Load the :class:`ReferrerPolicy` class to use for *policy*. + + *policy* may be any of the following: + + - A standard policy name, e.g. ``"no-referrer"``, + ``"origin-when-cross-origin"``, etc. + + - The special ``"scrapy-default"`` policy. + + - The import path of a :class:`ReferrerPolicy` subclass, e.g. + ``"scrapy.spidermiddlewares.referer.NoReferrerPolicy"`` or + ``"myproject.policies.CustomReferrerPolicy"``. + + If *warning_only* is ``False`` (default) and *policy* cannot be turned + into a :class:`ReferrerPolicy` subclass, a :exc:`RuntimeError` is + raised. If *warning_only* is ``True``, a warning is logged and ``None`` + is returned instead. + + If *allow_import_path* is ``False`` (default), import paths are not + allowed, resulting in :exc:`RuntimeError` or ``None``. If ``True``, + they are allowed. Use ``True`` only if you trust the source of the + *policy* value. + """ + if allow_import_path: + try: + return cast("type[ReferrerPolicy]", load_object(policy)) + except ValueError: + pass + policy_names = [ + policy_name.strip() for policy_name in policy.lower().split(",") + ] + # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header + for policy_name in policy_names[::-1]: + if policy_name in self.policies: + return self.policies[policy_name] + msg = f"Could not load referrer policy {policy!r}" + if not allow_import_path and _looks_like_import_path(policy): + msg += " (import paths from the response Referrer-Policy header are not allowed)" + if not warning_only: + raise RuntimeError(msg) + warnings.warn(msg, RuntimeWarning, stacklevel=2) + return None + def get_processed_request( self, request: Request, response: Response | None ) -> Request | None: @@ -383,25 +437,3 @@ class RefererMiddleware(BaseSpiderMiddleware): if referrer is not None: request.headers.setdefault("Referer", referrer) return request - - def request_scheduled(self, request: Request, spider: Spider) -> None: - # check redirected request to patch "Referer" header if necessary - redirected_urls = request.meta.get("redirect_urls", []) - if redirected_urls: - request_referrer = request.headers.get("Referer") - # we don't patch the referrer value if there is none - if request_referrer is not None: - # the request's referrer header value acts as a surrogate - # for the parent response URL - # - # Note: if the 3xx response contained a Referrer-Policy header, - # the information is not available using this hook - parent_url = safe_url_string(request_referrer) - policy_referrer = self.policy(parent_url, request).referrer( - parent_url, request.url - ) - if policy_referrer != request_referrer.decode("latin1"): - if policy_referrer is None: - request.headers.pop("Referer") - else: - request.headers["Referer"] = policy_referrer diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 018e510c5..de527d04f 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,17 +7,15 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -import warnings from typing import TYPE_CHECKING, Any, cast from scrapy import signals -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterable + from collections.abc import AsyncIterator from twisted.internet.defer import Deferred @@ -26,7 +24,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.http.request import CallbackT - from scrapy.settings import BaseSettings, _SettingsKey + from scrapy.settings import BaseSettings from scrapy.utils.log import SpiderLoggerAdapter @@ -39,7 +37,7 @@ class Spider(object_ref): """ name: str - custom_settings: dict[_SettingsKey, Any] | None = None + custom_settings: dict[str, Any] | None = None #: Start URLs. See :meth:`start`. start_urls: list[str] @@ -127,30 +125,6 @@ class Spider(object_ref): .. seealso:: :ref:`start-requests` """ - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" - ) - for item_or_request in self.start_requests(): - yield item_or_request - - def start_requests(self) -> Iterable[Any]: - warnings.warn( - ( - "The Spider.start_requests() method is deprecated, use " - "Spider.start() instead. If you are calling " - "super().start_requests() from a Spider.start() override, " - "iterate super().start() instead." - ), - ScrapyDeprecationWarning, - stacklevel=2, - ) - if not self.start_urls and hasattr(self, "start_url"): - raise AttributeError( - "Crawling could not start: 'start_urls' not found " - "or empty (but found 'start_url' attribute instead, " - "did you miss an 's'?)" - ) for url in self.start_urls: yield Request(url, dont_filter=True) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 89421cbe4..373c0b8b5 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -12,6 +12,7 @@ import warnings from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.link import Link from scrapy.linkextractors import LinkExtractor @@ -46,7 +47,9 @@ def _identity_process_request(request: Request, response: Response) -> Request | return request -def _get_method(method: Callable | str | None, spider: Spider) -> Callable | None: +def _get_method( + method: Callable[..., Any] | str | None, spider: Spider +) -> Callable[..., Any] | None: if callable(method): return method if isinstance(method, str): @@ -102,11 +105,13 @@ class CrawlSpider(Spider): self._compile_rules() if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"): warnings.warn( - f"The CrawlSpider._parse_response method, which the " + "The CrawlSpider._parse_response method, which the " f"{global_object_name(self.__class__)} class overrides, is " - f"deprecated: it will be removed in future Scrapy releases. " - f"Please override the CrawlSpider.parse_with_rules method " - f"instead." + "deprecated: it will be removed in future Scrapy releases. " + "Please override the CrawlSpider.parse_with_rules method " + "instead.", + ScrapyDeprecationWarning, + stacklevel=2, ) def _parse(self, response: Response, **kwargs: Any) -> Any: @@ -118,7 +123,7 @@ class CrawlSpider(Spider): ) def parse_start_url(self, response: Response, **kwargs: Any) -> Any: - return [] + return () def process_results( self, response: Response, results: Iterable[Any] @@ -195,6 +200,7 @@ class CrawlSpider(Spider): "The CrawlSpider._parse_response method is deprecated: " "it will be removed in future Scrapy releases. " "Please use the CrawlSpider.parse_with_rules method instead.", + ScrapyDeprecationWarning, stacklevel=2, ) return self.parse_with_rules(response, callback, cb_kwargs, follow) @@ -209,11 +215,19 @@ class CrawlSpider(Spider): def _compile_rules(self) -> None: self._rules = [] for rule in self.rules: - self._rules.append(copy.copy(rule)) - self._rules[-1]._compile(self) + copied_rule = copy.copy(rule) + copied_rule._compile(self) + self._rules.append(copied_rule) @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS") + if not spider._follow_links: + warnings.warn( + "The CRAWLSPIDER_FOLLOW_LINKS setting is deprecated." + " You can set follow=False in your rules to achieve the same effect.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) return spider diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py deleted file mode 100644 index 957bfffd3..000000000 --- a/scrapy/spiders/init.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -import warnings -from typing import TYPE_CHECKING, Any, cast - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.spiders import Spider -from scrapy.utils.spider import iterate_spider_output - -if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterable - - from scrapy import Request - from scrapy.http import Response - - -class InitSpider(Spider): - """Base Spider with initialization facilities - - .. warning:: This class is deprecated. Copy its code into your project if needed. - It will be removed in a future Scrapy version. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - warnings.warn( - "InitSpider is deprecated. Copy its code from Scrapy's source if needed. " - "Will be removed in a future version.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - - async def start(self) -> AsyncIterator[Any]: - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$" - ) - for item_or_request in self.start_requests(): - yield item_or_request - - def start_requests(self) -> Iterable[Request]: - self._postinit_reqs: Iterable[Request] = super().start_requests() - return cast("Iterable[Request]", iterate_spider_output(self.init_request())) - - def initialized(self, response: Response | None = None) -> Any: - """This method must be set as the callback of your last initialization - request. See self.init_request() docstring for more info. - """ - return self.__dict__.pop("_postinit_reqs") - - def init_request(self) -> Any: - """This function should return one initialization request, with the - self.initialized method as callback. When the self.initialized method - is called this spider is considered initialized. If you need to perform - several requests for initializing your spider, you can do so by using - different callbacks. The only requirement is that the final callback - (of the last initialization request) must be self.initialized. - - The default implementation calls self.initialized immediately, and - means that no initialization is needed. This method should be - overridden only when you need to perform requests to initialize your - spider - """ - return self.initialized() diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 2813a32a0..2a80b8d24 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -49,15 +49,11 @@ class SitemapSpider(Spider): self._cbs: list[tuple[re.Pattern[str], CallbackT]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): - c = cast("CallbackT", getattr(self, c)) + c = cast("CallbackT", getattr(self, c)) # noqa: PLW2901 self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] async def start(self) -> AsyncIterator[Any]: - for item_or_request in self.start_requests(): - yield item_or_request - - def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) @@ -72,31 +68,53 @@ class SitemapSpider(Spider): def _parse_sitemap(self, response: Response) -> Iterable[Request]: if response.url.endswith("/robots.txt"): - for url in sitemap_urls_from_robots(response.text, base_url=response.url): - yield Request(url, callback=self._parse_sitemap) - else: - body = self._get_sitemap_body(response) - if body is None: - logger.warning( - "Ignoring invalid sitemap: %(response)s", - {"response": response}, - extra={"spider": self}, - ) - return + urls = list(sitemap_urls_from_robots(response.body, base_url=response.url)) + return (Request(url, callback=self._parse_sitemap) for url in urls) - s = Sitemap(body) - it = self.sitemap_filter(s) + body = self._get_sitemap_body(response) + if not body: + logger.warning( + "Ignoring invalid sitemap: %(response)s", + {"response": response}, + extra={"spider": self}, + ) + return () - if s.type == "sitemapindex": - for loc in iterloc(it, self.sitemap_alternate_links): - if any(x.search(loc) for x in self._follow): - yield Request(loc, callback=self._parse_sitemap) - elif s.type == "urlset": - for loc in iterloc(it, self.sitemap_alternate_links): - for r, c in self._cbs: - if r.search(loc): - yield Request(loc, callback=c) - break + s = Sitemap(body) + + if s.type == "sitemapindex": + urls = list(self._get_urls_from_sitemapindex(self.sitemap_filter(s))) + return (Request(loc, callback=self._parse_sitemap) for loc in urls) + + if s.type == "urlset": + url_callback_pairs = list( + self._get_urls_and_callbacks_from_urlset(self.sitemap_filter(s)) + ) + return (Request(loc, callback=c) for loc, c in url_callback_pairs) + + logger.warning( + "Ignoring invalid sitemap: %(response)s", + {"response": response}, + extra={"spider": self}, + ) + + return () + + def _get_urls_from_sitemapindex( + self, it: Iterable[dict[str, Any]] + ) -> Iterable[str]: + for loc in iterloc(it, self.sitemap_alternate_links): + if any(x.search(loc) for x in self._follow): + yield loc + + def _get_urls_and_callbacks_from_urlset( + self, it: Iterable[dict[str, Any]] + ) -> Iterable[tuple[str, CallbackT]]: + for loc in iterloc(it, self.sitemap_alternate_links): + for r, c in self._cbs: + if r.search(loc): + yield loc, c + break def _get_sitemap_body(self, response: Response) -> bytes | None: """Return the sitemap body contained in the given response, @@ -140,8 +158,9 @@ def regex(x: re.Pattern[str] | str) -> re.Pattern[str]: def iterloc(it: Iterable[dict[str, Any]], alt: bool = False) -> Iterable[str]: for d in it: - yield d["loc"] + if loc := d["loc"]: + yield loc # Also consider alternate URLs (xhtml:link rel="alternate") - if alt and "alternate" in d: - yield from d["alternate"] + if alt and (alt_list := d.get("alternate")): + yield from alt_list diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 7007cd4b8..e06e38e23 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] - def __init__(self, path: str | PathLike, *args: Any, **kwargs: Any): + def __init__(self, path: str | PathLike[str], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index 88a18331c..e7d525f36 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -3,10 +3,11 @@ # See documentation in: # https://docs.scrapy.org/en/latest/topics/items.html -import scrapy +from dataclasses import dataclass -class ${ProjectName}Item(scrapy.Item): +@dataclass +class ${ProjectName}Item: # define the fields for your item here like: - # name = scrapy.Field() + # name: str | None = None pass diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py new file mode 100644 index 000000000..17957aba4 --- /dev/null +++ b/scrapy/utils/_deps_compat.py @@ -0,0 +1,28 @@ +import sys + +from OpenSSL import __version__ as PYOPENSSL_VERSION_STRING +from packaging.version import Version +from twisted import version as TWISTED_VERSION +from twisted.python.versions import Version as TxVersion +from w3lib import __version__ as W3LIB_VERSION_STRING + +# improved urllib.robotparser, https://github.com/python/cpython/pull/149374 +STDLIB_IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) or ( + (3, 13, 14) <= sys.version_info < (3, 14) +) + +TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0) +# changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506 +TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0) +# lowerMaximumSecurityTo off-by-1, https://github.com/twisted/twisted/issues/10232 +TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0) + +PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING) +# pyOpenSSL X.509 APIs are deprecated and cryptography-based ones are preferred +PYOPENSSL_X509_DEPRECATED = PYOPENSSL_VERSION >= Version("24.3.0") +# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable +PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0") + +W3LIB_VERSION = Version(W3LIB_VERSION_STRING) +# safe_url_string() strips the input, https://github.com/scrapy/w3lib/pull/207 +W3LIB_STRIPS_URLS = W3LIB_VERSION >= Version("2.1.1") diff --git a/scrapy/utils/_download_handlers.py b/scrapy/utils/_download_handlers.py index 8b0f0fa53..23b715ac6 100644 --- a/scrapy/utils/_download_handlers.py +++ b/scrapy/utils/_download_handlers.py @@ -2,8 +2,8 @@ from __future__ import annotations -from abc import ABC from contextlib import contextmanager +from http.cookiejar import CookieJar from typing import TYPE_CHECKING, Any from twisted.internet.defer import CancelledError @@ -15,7 +15,6 @@ from twisted.web.client import ResponseFailed from twisted.web.error import SchemeNotSupported from scrapy import responsetypes -from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.exceptions import ( CannotResolveHostError, DownloadCancelledError, @@ -29,26 +28,24 @@ from scrapy.utils.log import logger if TYPE_CHECKING: from collections.abc import Iterator + from http.client import HTTPResponse + from http.cookiejar import Cookie from ipaddress import IPv4Address, IPv6Address - - from twisted.internet.ssl import Certificate + from urllib.request import Request as ULRequest from scrapy import Request from scrapy.crawler import Crawler from scrapy.http import Headers, Response -class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): - """Base class for built-in HTTP download handlers.""" +class NullCookieJar(CookieJar): # pragma: no cover + """A CookieJar that rejects all cookies.""" - def __init__(self, crawler: Crawler): - super().__init__(crawler) - self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE") - self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE") - self._fail_on_dataloss: bool = crawler.settings.getbool( - "DOWNLOAD_FAIL_ON_DATALOSS" - ) - self._fail_on_dataloss_warned: bool = False + def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None: + pass + + def set_cookie(self, cookie: Cookie) -> None: + pass @contextmanager @@ -100,7 +97,7 @@ def make_response( headers: Headers, body: bytes = b"", flags: list[str] | None = None, - certificate: Certificate | None = None, + certificate: Any = None, ip_address: IPv4Address | IPv6Address | None = None, protocol: str | None = None, stop_download: StopDownload | None = None, @@ -146,3 +143,11 @@ def get_dataloss_msg(url: str) -> str: f"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" f" -- This message won't be shown in further requests" ) + + +def normalize_bind_address( + value: str | tuple[str, int] | None, +) -> tuple[str, int] | None: + if isinstance(value, str): + return (value, 0) + return value diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 9ec090995..44604c0fe 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar from twisted.internet.defer import Deferred from twisted.internet.task import LoopingCall +from twisted.internet.threads import deferToThread from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed @@ -64,7 +65,7 @@ def is_asyncio_available() -> bool: calling it from code such as spiders and Scrapy components, if Scrapy is run using one of the supported ways). - .. versionchanged:: VERSION + .. versionchanged:: 2.15.0 This function now also returns ``True`` if there is a running asyncio loop, even if no Twisted reactor is installed. """ @@ -131,20 +132,23 @@ async def _parallel_asyncio( class AsyncioLoopingCall: """A simple implementation of a periodic call using asyncio, keeping - some API and behavior compatibility with the Twisted ``LoopingCall``. + some API and behavior compatibility with + :class:`~twisted.internet.task.LoopingCall`. The function is called every *interval* seconds, independent of the finish time of the previous call. If the function is still running when it's time to call it again, calls are skipped until the function finishes. The function must not return a coroutine or a ``Deferred``. + + .. versionadded:: 2.14.0 """ def __init__(self, func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs): self._func: Callable[_P, _T] = func self._args: tuple[Any, ...] = args self._kwargs: dict[str, Any] = kwargs - self._task: asyncio.Task | None = None + self._task: asyncio.Task[None] | None = None self.interval: float | None = None self._start_time: float | None = None @@ -168,7 +172,7 @@ class AsyncioLoopingCall: raise ValueError("Interval must be greater than 0") self.interval = interval - self._start_time = time.time() + self._start_time = time.monotonic() if now: self._call() loop = asyncio.get_event_loop() @@ -178,7 +182,7 @@ class AsyncioLoopingCall: """Return the time to sleep until the next call.""" assert self.interval is not None assert self._start_time is not None - now = time.time() + now = time.monotonic() running_for = now - self._start_time return self.interval - (running_for % self.interval) @@ -215,8 +219,12 @@ def create_looping_call( ) -> AsyncioLoopingCall | LoopingCall: """Create an instance of a looping call class. - This creates an instance of :class:`AsyncioLoopingCall` or - :class:`LoopingCall`, depending on whether asyncio support is available. + This creates an instance of + :class:`~scrapy.utils.asyncio.AsyncioLoopingCall` or + :class:`~twisted.internet.task.LoopingCall`, depending on whether asyncio + support is available. + + .. versionadded:: 2.14.0 """ if is_asyncio_available(): return AsyncioLoopingCall(func, *args, **kwargs) @@ -228,8 +236,11 @@ def call_later( ) -> CallLaterResult: """Schedule a function to be called after a delay. - This uses either ``loop.call_later()`` or ``reactor.callLater()``, depending - on whether asyncio support is available. + This uses either :meth:`asyncio.loop.call_later` or + :meth:`reactor.callLater() `, + depending on whether asyncio support is available. + + .. versionadded:: 2.14.0 """ if is_asyncio_available(): loop = asyncio.get_event_loop() @@ -248,6 +259,8 @@ class CallLaterResult: no ``active()`` (as there is no such public API in :class:`asyncio.TimerHandle`) but ``cancel()`` can be called on already called or cancelled instances. + + .. versionadded:: 2.14.0 """ _timer_handle: asyncio.TimerHandle | None = None @@ -278,3 +291,23 @@ class CallLaterResult: elif self._delayed_call and self._delayed_call.active(): self._delayed_call.cancel() self._delayed_call = None + + +async def run_in_thread( + func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs +) -> _T: + """Call a function in a thread and return its result as a coroutine. + + This uses either :func:`asyncio.to_thread` or + :func:`twisted.internet.threads.deferToThread`, depending on whether + asyncio support is available. + + .. versionadded:: 2.15.0 + """ + if is_asyncio_available(): + return await asyncio.to_thread(func, *args, **kwargs) + + # circular import + from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 + + return await maybe_deferred_to_future(deferToThread(func, *args, **kwargs)) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 6ac5e66b2..403cd54a8 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -9,7 +9,7 @@ from twisted.web.server import Request, Site class Root(Resource): isLeaf = True - def getChild(self, name: str, request: Request) -> Resource: + def getChild(self, path: str, request: Request) -> Resource: return self def render(self, request: Request) -> bytes: @@ -30,13 +30,13 @@ class Root(Resource): def _getarg( request: Request, name: bytes, default: Any = None, type_: type = str ) -> Any: - return type_(request.args[name][0]) if name in request.args else default # type: ignore[index,operator] + return type_(request.args[name][0]) if name in request.args else default if __name__ == "__main__": from twisted.internet import reactor - root = Root() + root = Root() # type: ignore[no-untyped-call] factory = Site(root) httpPort = reactor.listenTCP(8998, Site(root)) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5869cf52e..4850b370b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -71,8 +71,8 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]: def closest_scrapy_cfg( - path: str | os.PathLike = ".", - prevpath: str | os.PathLike | None = None, + path: str | os.PathLike[str] = ".", + prevpath: str | os.PathLike[str] | None = None, ) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 87cfd3c28..93fbb9033 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -13,7 +13,7 @@ KnownShellsT = dict[str, Callable[..., EmbedFuncT]] def _embed_ipython_shell( - namespace: dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start an IPython Shell""" try: @@ -28,8 +28,8 @@ def _embed_ipython_shell( ) @wraps(_embed_ipython_shell) - def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: - config = load_default_config() + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: + config = load_default_config() # type: ignore[no-untyped-call] # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports # and clear the instance to always have the fresh env @@ -44,26 +44,26 @@ def _embed_ipython_shell( def _embed_bpython_shell( - namespace: dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start a bpython shell""" import bpython # noqa: PLC0415 @wraps(_embed_bpython_shell) - def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: bpython.embed(locals_=namespace, banner=banner) return wrapper def _embed_ptpython_shell( - namespace: dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start a ptpython shell""" import ptpython.repl # noqa: PLC0415 # pylint: disable=import-error @wraps(_embed_ptpython_shell) - def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: print(banner) ptpython.repl.embed(locals=namespace) @@ -71,7 +71,7 @@ def _embed_ptpython_shell( def _embed_standard_shell( - namespace: dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start a standard python shell""" try: # readline module is only available on unix systems @@ -84,7 +84,7 @@ def _embed_standard_shell( readline.parse_and_bind("tab:complete") # type: ignore[attr-defined,unused-ignore] @wraps(_embed_standard_shell) - def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: code.interact(banner=banner, local=namespace) return wrapper diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index a40ee8997..646335fb1 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -106,7 +106,7 @@ def curl_to_request_kwargs( if argv: msg = f"Unrecognized options: {', '.join(argv)}" if ignore_unknown_options: - warnings.warn(msg) + warnings.warn(msg, stacklevel=2) else: raise ValueError(msg) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 64d2cde1b..4e65c062e 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -13,7 +13,7 @@ import warnings import weakref from collections import OrderedDict from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, TypeVar +from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast from scrapy.exceptions import ScrapyDeprecationWarning @@ -28,7 +28,7 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -class CaselessDict(dict): +class CaselessDict(dict): # type: ignore[type-arg] __slots__ = () def __new__(cls, *args: Any, **kwargs: Any) -> Self: @@ -99,20 +99,20 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class CaseInsensitiveDict(collections.UserDict): +class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]): """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. """ def __init__(self, *args: Any, **kwargs: Any) -> None: - self._keys: dict = {} + self._keys: dict[str | bytes, Any] = {} super().__init__(*args, **kwargs) - def __getitem__(self, key: AnyStr) -> Any: + def __getitem__(self, key: str | bytes) -> Any: normalized_key = self._normkey(key) return super().__getitem__(self._keys[normalized_key.lower()]) - def __setitem__(self, key: AnyStr, value: Any) -> None: + def __setitem__(self, key: str | bytes, value: Any) -> None: normalized_key = self._normkey(key) try: lower_key = self._keys[normalized_key.lower()] @@ -122,19 +122,19 @@ class CaseInsensitiveDict(collections.UserDict): super().__setitem__(normalized_key, self._normvalue(value)) self._keys[normalized_key.lower()] = normalized_key - def __delitem__(self, key: AnyStr) -> None: + def __delitem__(self, key: str | bytes) -> None: normalized_key = self._normkey(key) stored_key = self._keys.pop(normalized_key.lower()) super().__delitem__(stored_key) - def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + def __contains__(self, key: str | bytes) -> bool: # type: ignore[override] normalized_key = self._normkey(key) return normalized_key.lower() in self._keys def __repr__(self) -> str: return f"<{self.__class__.__name__}: {super().__repr__()}>" - def _normkey(self, key: AnyStr) -> AnyStr: + def _normkey(self, key: str | bytes) -> str | bytes: return key def _normvalue(self, value: Any) -> Any: @@ -158,7 +158,7 @@ class LocalCache(OrderedDict[_KT, _VT]): super().__setitem__(key, value) -class LocalWeakReferencedCache(weakref.WeakKeyDictionary): +class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]): """ A weakref.WeakKeyDictionary implementation that uses LocalCache as its underlying data structure, making it ordered and capable of being size-limited. @@ -172,16 +172,16 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): def __init__(self, limit: int | None = None): super().__init__() - self.data: LocalCache = LocalCache(limit=limit) + self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit) - def __setitem__(self, key: _KT, value: _VT) -> None: + def __setitem__(self, key: _KT, value: _VT | None) -> None: # if raised, key is not weak-referenceable, skip caching with contextlib.suppress(TypeError): super().__setitem__(key, value) def __getitem__(self, key: _KT) -> _VT | None: try: - return super().__getitem__(key) + return cast("_VT", super().__getitem__(key)) except (TypeError, KeyError): return None # key is either not weak-referenceable or not cached diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 044d1d4c7..a5bb6fa24 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -3,12 +3,13 @@ from __future__ import annotations import inspect import warnings from functools import wraps -from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, overload +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast, overload from twisted.internet.defer import Deferred, maybeDeferred -from twisted.internet.threads import deferToThread from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.asyncio import run_in_thread +from scrapy.utils.defer import deferred_from_coro if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine @@ -59,12 +60,15 @@ def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: # pragma: no def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: """Decorator to call a function in a thread and return a deferred with the - result + result. + + .. versionchanged:: 2.15.0 + Now uses :func:`asyncio.to_thread` if the asyncio support is available. """ @wraps(func) def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]: - return deferToThread(func, *a, **kw) + return deferred_from_coro(run_in_thread(func, *a, **kw)) return wrapped @@ -111,7 +115,7 @@ def _warn_spider_arg( @wraps(func) async def async_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T: check_args(*args, **kwargs) - return await func(*args, **kwargs) + return cast("_T", await func(*args, **kwargs)) return async_inner diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 1acb576b4..29a34d4ef 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -8,6 +8,7 @@ import asyncio import inspect import warnings from asyncio import Future +from collections import deque from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps from typing import ( @@ -172,7 +173,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator, Generic[_T]): +class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_T]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more @@ -230,7 +231,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): self.callable_args: tuple[Any, ...] = callable_args self.callable_kwargs: dict[str, Any] = callable_kwargs self.finished: bool = False - self.waiting_deferreds: list[Deferred[Any]] = [] + self.waiting_deferreds: deque[Deferred[Any]] = deque() self.anext_deferred: Deferred[_T] | None = None def _callback(self, result: _T) -> None: @@ -241,7 +242,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): callable_result = self.callable( result, *self.callable_args, **self.callable_kwargs ) - d = self.waiting_deferreds.pop(0) + d = self.waiting_deferreds.popleft() if isinstance(callable_result, Deferred): callable_result.chainDeferred(d) else: @@ -261,7 +262,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) + self.anext_deferred = deferred_from_coro(anext(self.aiterator)) self.anext_deferred.addCallbacks(self._callback, self._errback) def __next__(self) -> Deferred[Any]: @@ -369,10 +370,10 @@ async def aiter_errback( """Wrap an async iterable calling an errback if an error is caught while iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`. """ - it = aiterable.__aiter__() + it = aiter(aiterable) while True: try: - yield await it.__anext__() + yield await anext(it) except StopAsyncIteration: break except Exception: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3bf6639c5..359f819d7 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -4,7 +4,7 @@ from __future__ import annotations import inspect import warnings -from typing import TYPE_CHECKING, Any, overload +from typing import TYPE_CHECKING, Any, cast, overload from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import get_func_args_dict @@ -68,7 +68,7 @@ def create_deprecated_class( def __new__( # pylint: disable=bad-classmethod-argument metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any] ) -> type: - cls = super().__new__(metacls, name, bases, clsdict_) + cls: type = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls @@ -100,7 +100,7 @@ def create_deprecated_class( # is the deprecated class itself - subclasses of the # deprecated class should not use custom `__subclasscheck__` # method. - return super().__subclasscheck__(sub) + return cast("bool", super().__subclasscheck__(sub)) if not inspect.isclass(sub): raise TypeError("issubclass() arg 1 must be a class") @@ -130,7 +130,7 @@ def create_deprecated_class( # deprecated class is in jinja2 template). __module__ attribute is not # important enough to raise an exception as users may be unable # to fix inspect.stack() errors. - warnings.warn(f"Error detecting parent module: {e!r}") + warnings.warn(f"Error detecting parent module: {e!r}", stacklevel=2) return deprecated_cls @@ -160,6 +160,7 @@ def update_classpath(path: Any) -> Any: warnings.warn( f"`{path}` class is deprecated, use `{new_path}` instead", ScrapyDeprecationWarning, + stacklevel=2, ) return new_path return path diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 102362506..09b67805d 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -12,7 +12,7 @@ from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.settings import Settings, _SettingsKey +from scrapy.settings import Settings from scrapy.utils.versions import get_versions if TYPE_CHECKING: @@ -89,7 +89,7 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Settings | dict[_SettingsKey, Any] | None = None, + settings: Settings | dict[str, Any] | None = None, install_root_handler: bool = True, ) -> None: """ @@ -260,7 +260,8 @@ def logformatter_adapter( return (level, message, args) -class SpiderLoggerAdapter(logging.LoggerAdapter): +# LoggerAdapter is only parameterized since Python 3.11 +class SpiderLoggerAdapter(logging.LoggerAdapter): # type: ignore[type-arg] def process( self, msg: str, kwargs: MutableMapping[str, Any] ) -> tuple[str, MutableMapping[str, Any]]: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 7994a2471..20e7cb381 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -13,7 +13,7 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import IO, TYPE_CHECKING, Any, TypeVar, cast +from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overload from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item @@ -28,9 +28,20 @@ if TYPE_CHECKING: _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes -T = TypeVar("T") +_ITER_T = TypeVar("_ITER_T", bound=dict[Any, Any] | Item | str | bytes) +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_P = ParamSpec("_P") +@overload +def arg_to_iter(arg: None) -> tuple[()]: ... +@overload +def arg_to_iter(arg: _ITER_T) -> Iterable[_ITER_T]: ... +@overload +def arg_to_iter(arg: Iterable[_T]) -> Iterable[_T]: ... +@overload +def arg_to_iter(arg: _T) -> Iterable[_T]: ... def arg_to_iter(arg: Any) -> Iterable[Any]: """Convert an argument to an iterable. The argument can be a None, single value, or an iterable. @@ -38,7 +49,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: Exception: if arg is a dict, [arg] will be returned """ if arg is None: - return [] + return () if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"): return cast("Iterable[Any]", arg) return [arg] @@ -64,7 +75,7 @@ def load_object(path: str | Callable[..., Any]) -> Any: try: dot = path.rindex(".") except ValueError: - raise ValueError(f"Error loading object '{path}': not a full path") + raise ValueError(f"Error loading object '{path}': not a full path") from None module, name = path[:dot], path[dot + 1 :] mod = import_module(module) @@ -72,41 +83,61 @@ def load_object(path: str | Callable[..., Any]) -> Any: try: obj = getattr(mod, name) except AttributeError: - raise NameError(f"Module '{module}' doesn't define any object named '{name}'") + raise NameError( + f"Module '{module}' doesn't define any object named '{name}'" + ) from None return obj -def walk_modules(path: str) -> list[ModuleType]: +def walk_modules_iter(path: str) -> Iterable[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that exception is thrown back. - For example: walk_modules('scrapy.utils') + For example: + >>> list(walk_modules_iter('scrapy.utils')) + [, ...] + >>> gen = walk_modules_iter('scrapy.utils.nonexistent') # error not raised until the generator is consumed + >>> list(gen) + Traceback (most recent call last): + ... + ModuleNotFoundError: No module named 'scrapy.utils.nonexistent' """ - mods: list[ModuleType] = [] mod = import_module(path) - mods.append(mod) + yield mod if hasattr(mod, "__path__"): for _, subpath, ispkg in iter_modules(mod.__path__): fullpath = path + "." + subpath if ispkg: - mods += walk_modules(fullpath) + yield from walk_modules_iter(fullpath) else: - submod = import_module(fullpath) - mods.append(submod) - return mods + yield import_module(fullpath) -def md5sum(file: IO[bytes]) -> str: - """Calculate the md5 checksum of a file-like object without reading its - whole content in memory. - - >>> from io import BytesIO - >>> md5sum(BytesIO(b'file content to hash')) - '784406af91dd5a54fbb9c84c2236595a' +def walk_modules(path: str) -> list[ModuleType]: # pragma: no cover """ + Loads a module and all its submodules from the given module path and + returns them. If *any* module throws an exception while importing, that + exception is thrown back. + """ + warnings.warn( + ( + "The scrapy.utils.misc.walk_modules function is deprecated and will be " + "removed in a future version of Scrapy. " + "Use scrapy.utils.misc.walk_modules_iter instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + + return list(walk_modules_iter(path)) + + +def md5sum(file: IO[bytes]) -> str: # pragma: no cover + """Calculate the md5 checksum of a file-like object without reading its + whole content in memory.""" warnings.warn( ( "The scrapy.utils.misc.md5sum function is deprecated and will be " @@ -126,12 +157,43 @@ def md5sum(file: IO[bytes]) -> str: def rel_has_nofollow(rel: str | None) -> bool: """Return True if link rel attribute has nofollow type""" - return rel is not None and "nofollow" in rel.replace(",", " ").split() + return rel is not None and "nofollow" in rel.lower().replace(",", " ").split() + + +class SupportsFromCrawler(Protocol[_T_co, _P]): + @classmethod + def from_crawler( + cls, crawler: Crawler, /, *args: _P.args, **kwargs: _P.kwargs + ) -> _T_co: ... + + +@overload +def build_from_crawler( + objcls: SupportsFromCrawler[_T_co, _P], + crawler: Crawler, + /, + *args: _P.args, + **kwargs: _P.kwargs, +) -> _T_co: ... + + +@overload +def build_from_crawler( + objcls: Callable[_P, _T_co], + crawler: Crawler, + /, + *args: _P.args, + **kwargs: _P.kwargs, +) -> _T_co: ... def build_from_crawler( - objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any -) -> T: + objcls: Any, + crawler: Crawler, + /, + *args: Any, + **kwargs: Any, +) -> Any: """Construct a class instance using its ``from_crawler()`` or ``__init__()`` constructor. .. versionadded:: 2.12 @@ -141,14 +203,14 @@ def build_from_crawler( Raises ``TypeError`` if the resulting instance is ``None``. """ if hasattr(objcls, "from_crawler"): - instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined] + instance = objcls.from_crawler(crawler, *args, **kwargs) method_name = "from_crawler" else: instance = objcls(*args, **kwargs) method_name = "__new__" if instance is None: raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return cast("T", instance) + return instance @contextmanager @@ -185,7 +247,14 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]: yield node -_generator_callbacks_cache = LocalWeakReferencedCache(limit=128) +_generator_callbacks_cache: LocalWeakReferencedCache[Callable[..., Any], bool] = ( + LocalWeakReferencedCache(limit=128) +) + + +def _returns_none(return_node: ast.Return) -> bool: + value = return_node.value + return value is None or (isinstance(value, ast.Constant) and value.value is None) def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noqa: A002 @@ -196,12 +265,6 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noq if callable in _generator_callbacks_cache: return bool(_generator_callbacks_cache[callable]) - def returns_none(return_node: ast.Return) -> bool: - value = return_node.value - return value is None or ( - isinstance(value, ast.Constant) and value.value is None - ) - if inspect.isgeneratorfunction(callable): func = callable while isinstance(func, partial): @@ -217,7 +280,7 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noq tree = ast.parse(code) for node in walk_callable(tree): - if isinstance(node, ast.Return) and not returns_none(node): + if isinstance(node, ast.Return) and not _returns_none(node): _generator_callbacks_cache[callable] = True return bool(_generator_callbacks_cache[callable]) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 0139720b7..3e75c7729 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -20,7 +20,8 @@ def inside_project() -> bool: import_module(scrapy_module) except ImportError as exc: warnings.warn( - f"Cannot import scrapy settings module {scrapy_module}: {exc}" + f"Cannot import scrapy settings module {scrapy_module}: {exc}", + stacklevel=2, ) else: return True diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 04da75a30..00cf818b7 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,12 +8,14 @@ import gc import inspect import re import sys +import warnings import weakref from collections.abc import AsyncIterator, Iterable, Mapping from functools import partial, wraps from itertools import chain from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: @@ -99,6 +101,16 @@ def to_bytes( return text.encode(encoding, errors) +def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]: + offset = len(text) + while True: + offset -= chunk_size * 1024 + if offset <= 0: + break + yield (text[offset:], offset) + yield (text, 0) + + def re_rsearch( pattern: str | Pattern[str], text: str, chunk_size: int = 1024 ) -> tuple[int, int] | None: @@ -115,19 +127,10 @@ def re_rsearch( the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter() -> Iterable[tuple[str, int]]: - offset = len(text) - while True: - offset -= chunk_size * 1024 - if offset <= 0: - break - yield (text[offset:], offset) - yield (text, 0) - if isinstance(pattern, str): pattern = re.compile(pattern) - for chunk, offset in _chunk_iter(): + for chunk, offset in _chunk_iter(text, chunk_size): matches = list(pattern.finditer(chunk)) if matches: start, end = matches[-1].span() @@ -293,12 +296,13 @@ else: gc.collect() -class MutableChain(Iterable[_T]): - """ - Thin wrapper around itertools.chain, allowing to add iterables "in-place" - """ - +class MutableChain(Iterable[_T]): # pragma: no cover def __init__(self, *args: Iterable[_T]): + warnings.warn( + "MutableChain is deprecated and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self.data: Iterator[_T] = chain.from_iterable(args) def extend(self, *iterables: Iterable[_T]) -> None: @@ -335,3 +339,33 @@ class MutableAsyncChain(AsyncIterator[_T]): async def __anext__(self) -> _T: return await self.data.__anext__() + + +def _looks_like_import_path(value: str) -> bool: + """Return True if **value** looks like a valid Python import path or False + otherwise.""" + if not value: + return False + if any(c.isspace() for c in value): + return False + allowed_chars = set( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_." + ) + if any(c not in allowed_chars for c in value): + return False + if value[0] == "." or value[-1] == ".": + return False + parts = value.split(".") + if any(part == "" for part in parts): + return False + return all(part.isidentifier() for part in parts) + + +def _iter_exc_causes(exc: BaseException) -> Iterable[BaseException]: + """Iterate over the exception causes/contexts.""" + seen: set[int] = set() + cur: BaseException | None = exc + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + yield cur + cur = cur.__cause__ or cur.__context__ diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 419c552bf..5499e3f48 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import sys +import warnings from contextlib import suppress from typing import TYPE_CHECKING, Any, Generic, ParamSpec, TypeVar from warnings import catch_warnings, filterwarnings @@ -32,13 +33,15 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: if len(portrange) > 2: raise ValueError(f"invalid portrange: {portrange}") + if len(portrange) == 2 and portrange[0] > portrange[1]: + raise ValueError(f"invalid portrange: {portrange}") if not portrange: - return reactor.listenTCP(0, factory, interface=host) + return reactor.listenTCP(0, factory, interface=host) # type: ignore[no-any-return] if len(portrange) == 1: - return reactor.listenTCP(portrange[0], factory, interface=host) + return reactor.listenTCP(portrange[0], factory, interface=host) # type: ignore[no-any-return] for x in range(portrange[0], portrange[1] + 1): try: - return reactor.listenTCP(x, factory, interface=host) + return reactor.listenTCP(x, factory, interface=host) # type: ignore[no-any-return] except error.CannotListenError: if x == portrange[1]: raise @@ -54,7 +57,7 @@ class CallLaterOnce(Generic[_T]): self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw self._call: CallLaterResult | None = None - self._deferreds: list[Deferred] = [] + self._deferreds: list[Deferred[None]] = [] def schedule(self, delay: float = 0) -> None: # circular import @@ -76,7 +79,7 @@ class CallLaterOnce(Generic[_T]): for d in self._deferreds: call_later(0, d.callback, None) - self._deferreds = [] + self._deferreds.clear() return result @@ -93,16 +96,19 @@ _asyncio_reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" def set_asyncio_event_loop_policy() -> None: - """The policy functions from asyncio often behave unexpectedly, - so we restrict their use to the absolutely essential case. - This should only be used to install the reactor. - """ - policy = asyncio.get_event_loop_policy() - if sys.platform == "win32" and not isinstance( - policy, asyncio.WindowsSelectorEventLoopPolicy - ): - policy = asyncio.WindowsSelectorEventLoopPolicy() - asyncio.set_event_loop_policy(policy) + """Needed due to https://github.com/twisted/twisted/issues/12527.""" + if sys.platform != "win32": + return + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"'asyncio\.(get_event_loop_policy|WindowsSelectorEventLoopPolicy)' is deprecated", + category=DeprecationWarning, + ) + policy = asyncio.get_event_loop_policy() + if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): + policy = asyncio.WindowsSelectorEventLoopPolicy() # pylint: disable=deprecated-class + asyncio.set_event_loop_policy(policy) def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> None: @@ -181,7 +187,7 @@ def verify_installed_reactor(reactor_path: str) -> None: def verify_installed_asyncio_event_loop(loop_path: str) -> None: - """Raise :exc:`RuntimeError` if the even loop of the installed + """Raise :exc:`RuntimeError` if the event loop of the installed :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` does not match the specified import path or if no reactor is installed.""" if not is_reactor_installed(): diff --git a/scrapy/utils/reactorless.py b/scrapy/utils/reactorless.py index cd60e6c7d..aca76951b 100644 --- a/scrapy/utils/reactorless.py +++ b/scrapy/utils/reactorless.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: def is_reactorless() -> bool: """Check if we are running in the reactorless mode, i.e. with - :setting:`TWISTED_ENABLED` set to ``False``. + :setting:`TWISTED_REACTOR_ENABLED` set to ``False``. As this checks the runtime state and not the setting itself, it can be wrong when executed very early, before the reactor and/or the asyncio event @@ -25,7 +25,7 @@ def is_reactorless() -> bool: :func:`scrapy.utils.asyncio.is_asyncio_available()`, it has the same limitations for detecting a running asyncio event loop as that one. - .. versionadded:: VERSION + .. versionadded:: 2.15.0 """ return is_asyncio_available() and not is_reactor_installed() diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 616ab661f..398403d90 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -28,7 +28,7 @@ if TYPE_CHECKING: _fingerprint_cache: WeakKeyDictionary[ - Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes] + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] ] = WeakKeyDictionary() @@ -71,8 +71,10 @@ def fingerprint( processed_include_headers = tuple( to_bytes(h.lower()) for h in sorted(include_headers) ) + verbatim_url = bool(request.meta.get("verbatim_url")) + effective_keep_fragments = keep_fragments and not verbatim_url cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (processed_include_headers, keep_fragments) + cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url) if cache_key not in cache: # To decode bytes reliably (JSON does not support bytes), regardless of # character encoding, we use bytes.hex() @@ -84,9 +86,13 @@ def fingerprint( header_value.hex() for header_value in request.headers.getlist(header) ] + if verbatim_url: + url = request.url + else: + url = canonicalize_url(request.url, keep_fragments=keep_fragments) fingerprint_data = { "method": to_unicode(request.method), - "url": canonicalize_url(request.url, keep_fragments=keep_fragments), + "url": url, "body": (request.body or b"").hex(), "headers": headers, } @@ -108,8 +114,9 @@ class RequestFingerprinter: (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url ` and the values of :attr:`request.method ` and :attr:`request.body - `. It then generates an `SHA1 - `_ hash. + `, unless :reqmeta:`verbatim_url` is true for that + request. It then generates an `SHA1 `_ + hash. """ @classmethod @@ -169,7 +176,13 @@ def _get_method(obj: Any, name: Any) -> Any: try: return getattr(obj, name) except AttributeError: - raise ValueError(f"Method {name!r} not found in: {obj}") + raise ValueError(f"Method {name!r} not found in: {obj}") from None + + +def _cookie_value_to_unicode(value: str | bytes | float) -> str: + if isinstance(value, bytes): + return value.decode() + return str(value) def request_to_curl(request: Request) -> str: @@ -195,7 +208,9 @@ def request_to_curl(request: Request) -> str: cookies = f"--cookie '{cookie}'" elif isinstance(request.cookies, list): cookie = "; ".join( - f"{next(iter(c.keys()))}={next(iter(c.values()))}" + f"{_cookie_value_to_unicode(c['name'])}={_cookie_value_to_unicode(c['value'])}" + if "name" in c and "value" in c + else f"{next(iter(c.keys()))}={next(iter(c.values()))}" for c in request.cookies ) cookies = f"--cookie '{cookie}'" diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 8b2417bfb..c068d9b1e 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -97,9 +97,8 @@ def open_in_browser( # XXX: this implementation is a bit dirty and could be improved body = response.body if isinstance(response, HtmlResponse): - if b"' + if b"' body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" elif isinstance(response, TextResponse): diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 997e8eba8..919f67240 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -7,6 +7,7 @@ import logging import warnings from collections.abc import Awaitable, Callable, Generator, Sequence from typing import Any as TypingAny +from typing import cast from pydispatch.dispatcher import ( Anonymous, @@ -124,13 +125,11 @@ def _send_catch_log_deferred( **named, ) d.addErrback(logerror, receiver) - # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html + d2: Deferred[tuple[TypingAny, TypingAny]] = d.addBoth( - lambda result: ( - receiver, # pylint: disable=cell-var-from-loop # noqa: B023 - result, - ) + lambda result, recv: (recv, result), receiver ) + dfds.append(d2) results = yield DeferredList(dfds) @@ -185,7 +184,9 @@ async def _send_catch_log_asyncio( handlers: list[Awaitable[TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - async def handler(receiver: Callable) -> TypingAny: + async def handler( + receiver: Callable[..., Any], + ) -> tuple[Callable[..., Any], TypingAny]: result: TypingAny try: result = await ensure_awaitable( @@ -208,7 +209,10 @@ async def _send_catch_log_asyncio( handlers.append(handler(receiver)) - return await asyncio.gather(*handlers, return_exceptions=True) + return cast( + "list[tuple[TypingAny, TypingAny]]", + await asyncio.gather(*handlers, return_exceptions=True), + ) def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index c0100ea62..1520a4ff0 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -7,11 +7,15 @@ SitemapSpider, its API is subject to change without notice. from __future__ import annotations +import warnings +from io import BytesIO, StringIO from typing import TYPE_CHECKING, Any from urllib.parse import urljoin import lxml.etree +from scrapy.exceptions import ScrapyDeprecationWarning + if TYPE_CHECKING: from collections.abc import Iterable, Iterator @@ -20,40 +24,112 @@ class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" + __slots__ = ("type", "xmliter") + def __init__(self, xmltext: str | bytes): - xmlp = lxml.etree.XMLParser( - recover=True, remove_comments=True, resolve_entities=False + if isinstance(xmltext, str): + warnings.warn( + "Passing `str` type as `xmltext` is deprecated, use `bytes`", + ScrapyDeprecationWarning, + stacklevel=2, + ) + xmltext = xmltext.encode() + + self.xmliter = lxml.etree.iterparse( + BytesIO(xmltext), + recover=True, + remove_comments=True, + resolve_entities=False, + remove_blank_text=True, + collect_ids=False, + remove_pis=True, + events=("start", "end"), ) - self._root = lxml.etree.fromstring(xmltext, parser=xmlp) - rt = self._root.tag - assert isinstance(rt, str) - self.type = rt.split("}", 1)[1] if "}" in rt else rt + _, root = next(self.xmliter) + self.type = self._get_tag_name(root) def __iter__(self) -> Iterator[dict[str, Any]]: - for elem in self._root.getchildren(): - d: dict[str, Any] = {} - for el in elem.getchildren(): - tag = el.tag - assert isinstance(tag, str) - name = tag.split("}", 1)[1] if "}" in tag else tag + for event, elem in self.xmliter: + if event == "start": + continue - if name == "link": - if "href" in el.attrib: - d.setdefault("alternate", []).append(el.get("href")) - else: - d[name] = el.text.strip() if el.text else "" + if self._get_tag_name(elem) not in {"url", "sitemap"}: + continue - if "loc" in d: + if d := self._process_sitemap_element(elem): yield d + def _process_sitemap_element( + self, elem: lxml.etree._Element + ) -> dict[str, Any] | None: + d: dict[str, Any] = {} + alternate: list[str] = [] + has_loc = False + + for el in elem: + try: + tag_name = self._get_tag_name(el) + if not tag_name: + continue + + if tag_name == "link": + if href := el.get("href"): + alternate.append(href) + else: + d[tag_name] = el.text.strip() if el.text else "" + if not has_loc and tag_name == "loc": + has_loc = True + finally: + el.clear() + elem.clear() + parent = elem.getparent() + if parent is not None: + while elem.getprevious() is not None: + del parent[0] + + if not has_loc: + return None + + if alternate: + d["alternate"] = alternate + + return d + + @staticmethod + def _get_tag_name(elem: lxml.etree._Element) -> str: + if TYPE_CHECKING: + assert isinstance(elem.tag, str) + _, _, localname = elem.tag.partition("}") + return localname or elem.tag + def sitemap_urls_from_robots( - robots_text: str, base_url: str | None = None + robots_text: str | bytes, + base_url: str | None = None, ) -> Iterable[str]: - """Return an iterator over all sitemap urls contained in the given - robots.txt file - """ - for line in robots_text.splitlines(): - if line.lstrip().lower().startswith("sitemap:"): - url = line.split(":", 1)[1].strip() + if isinstance(robots_text, bytes): + for line in BytesIO(robots_text): + if line.lstrip()[:8].lower() == b"sitemap:": + try: + url = line.partition(b":")[2].strip().decode() + except UnicodeDecodeError: + continue + yield urljoin(base_url or "", url) + + else: + warnings.warn( + "Passing `str` type as `robots_text` is deprecated, use `bytes`", + ScrapyDeprecationWarning, + stacklevel=2, + ) + yield from _sitemap_urls_from_robots_str(robots_text, base_url) + + +def _sitemap_urls_from_robots_str( + robots_text: str, + base_url: str | None = None, +) -> Iterable[str]: + for line in StringIO(robots_text): + if line.lstrip()[:8].lower() == "sitemap:": + url = line.partition(":")[2].strip() yield urljoin(base_url or "", url) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 75d6c9bb0..9f43df1ce 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -41,11 +41,10 @@ def iterate_spider_output( ) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[_T]: if inspect.isasyncgen(result): return result + d: Deferred[_T] = deferred_from_coro(result) if inspect.iscoroutine(result): - d = deferred_from_coro(result) - d.addCallback(iterate_spider_output) - return d - return arg_to_iter(deferred_from_coro(result)) + return d.addCallback(iterate_spider_output) + return arg_to_iter(d) def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]: diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 0062da52b..828a99e23 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -2,30 +2,64 @@ from __future__ import annotations import logging import ssl -from typing import TYPE_CHECKING, Any +import warnings +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version +from twisted.internet.ssl import CertificateOptions, TLSVersion +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils._deps_compat import ( + PYOPENSSL_X509_DEPRECATED, + TWISTED_TLS_LIMITS_OFFBY1, +) from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Callable + from OpenSSL.crypto import X509Name from scrapy.settings import BaseSettings logger = logging.getLogger(__name__) +_T = TypeVar("_T") + + +# common + + +def _get_tls_version_limit( + settings: BaseSettings, setting_name: str, converter: Callable[[str], _T] +) -> _T | None: + setting: str | None = settings[setting_name] + if setting is None: + return None + try: + return converter(setting) + except Exception as ex: + raise ValueError(f"Unknown {setting_name} value: {setting}") from ex + + +def _get_tls_version_limits( + settings: BaseSettings, converter: Callable[[str], _T] +) -> tuple[_T | None, _T | None]: + return ( + _get_tls_version_limit(settings, "DOWNLOAD_TLS_MIN_VERSION", converter), + _get_tls_version_limit(settings, "DOWNLOAD_TLS_MAX_VERSION", converter), + ) + # stdlib ssl module utils -# possible documented values for DOWNLOADER_CLIENT_TLS_METHOD -_STDLIB_PROTOCOL_MAP = { - "TLS": ssl.PROTOCOL_TLS_CLIENT, - "TLSv1.0": ssl.PROTOCOL_TLSv1, - "TLSv1.1": ssl.PROTOCOL_TLSv1_1, - "TLSv1.2": ssl.PROTOCOL_TLSv1_2, +_STDLIB_VERSION_MAP: dict[str, ssl.TLSVersion] = { + "TLSv1.0": ssl.TLSVersion.TLSv1, + "TLSv1.1": ssl.TLSVersion.TLSv1_1, + "TLSv1.2": ssl.TLSVersion.TLSv1_2, + "TLSv1.3": ssl.TLSVersion.TLSv1_3, } @@ -35,19 +69,41 @@ def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext: It's intended to be used in an HTTPS download handler. """ - method_setting: str = settings["DOWNLOADER_CLIENT_TLS_METHOD"] - if method_setting not in _STDLIB_PROTOCOL_MAP: - raise ValueError(f"Unsupported TLS method: {method_setting}") + tls_min_ver, tls_max_ver = _get_tls_version_limits( + settings, _STDLIB_VERSION_MAP.__getitem__ + ) ciphers_setting: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + verify_setting = settings.getbool("DOWNLOAD_VERIFY_CERTIFICATES") - ctx = ssl.SSLContext(_STDLIB_PROTOCOL_MAP[method_setting]) - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + if verify_setting: + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_default_certs() + else: + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + if tls_min_ver is not None: + ctx.minimum_version = tls_min_ver + if tls_max_ver is not None: + ctx.maximum_version = tls_max_ver if ciphers_setting: ctx.set_ciphers(ciphers_setting) return ctx +def _make_insecure_ssl_ctx() -> ssl.SSLContext: + """Create an SSL context that doesn't verify certificates. + + Compared to :func:`~scrapy.utils.ssl._make_ssl_context` this is much more + simple. + """ + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None: cipher = sslobj.cipher() logger.debug( @@ -55,30 +111,52 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None: f" using protocol {sslobj.version()}," f" cipher {cipher[0] if cipher else None}" ) - # The peer certificate is unavailable on SSLObject unless peer - # certificate verification is enabled, which we don't want. + if cert := sslobj.getpeercert(): + # Not available without certificate verification + logger.debug( + f'SSL connection certificate: issuer "{cert["issuer"]}", subject "{cert["subject"]}"' + ) # pyOpenSSL utils -def ffi_buf_to_string(buf: Any) -> str: +def _ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) -def x509name_to_string(x509name: X509Name) -> str: +def ffi_buf_to_string(buf: Any) -> str: # pragma: no cover + warnings.warn( + "ffi_buf_to_string() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return ffi_buf_to_string(buf) + + +def _x509name_to_string(x509name: X509Name) -> str: # from OpenSSL.crypto.X509Name.__repr__ + # only used on pyOpenSSL < 24.3.0 result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512) pyOpenSSLutil.lib.X509_NAME_oneline( x509name._name, result_buffer, len(result_buffer) ) - return ffi_buf_to_string(result_buffer) + return _ffi_buf_to_string(result_buffer) -def get_temp_key_info(ssl_object: Any) -> str | None: +def x509name_to_string(x509name: X509Name) -> str: # pragma: no cover + warnings.warn( + "x509name_to_string() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _x509name_to_string(x509name) + + +def _get_temp_key_info(ssl_object: Any) -> str | None: # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"): - # removed in cryptography 40.0.0 + # removed in cryptography 40.0.0 (required starting from pyOpenSSL 23.1.0) return None temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p): @@ -103,14 +181,88 @@ def get_temp_key_info(ssl_object: Any) -> str | None: cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid) if cname == pyOpenSSLutil.ffi.NULL: cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid) - key_info.append(ffi_buf_to_string(cname)) + key_info.append(_ffi_buf_to_string(cname)) else: - key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) + key_info.append(_ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits") return ", ".join(key_info) +def get_temp_key_info(ssl_object: Any) -> str | None: # pragma: no cover + warnings.warn( + "get_temp_key_info() is deprecated. It's also a no-op with cryptography 40.0.0+.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _get_temp_key_info(ssl_object) + + def get_openssl_version() -> str: system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) system_openssl = system_openssl_bytes.decode("ascii", errors="replace") return f"{OpenSSL.version.__version__} ({system_openssl})" + + +def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection) -> None: + logger.debug( + "SSL connection to %s using protocol %s, cipher %s", + hostname, + connection.get_protocol_version_name(), + connection.get_cipher_name(), + ) + if PYOPENSSL_X509_DEPRECATED: + if server_cert := connection.get_peer_certificate(as_cryptography=True): + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + server_cert.issuer.rfc4514_string(), + server_cert.subject.rfc4514_string(), + ) + else: # noqa: PLR5501 + if server_cert_pyopenssl := connection.get_peer_certificate(): + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + _x509name_to_string(server_cert_pyopenssl.get_issuer()), + _x509name_to_string(server_cert_pyopenssl.get_subject()), + ) + key_info = _get_temp_key_info(connection._ssl) + if key_info: + logger.debug("SSL temp key: %s", key_info) + + +# Twisted-specific + + +class _CertificateOptionsVersionKwargs(TypedDict, total=False): + lowerMaximumSecurityTo: TLSVersion + insecurelyLowerMinimumTo: TLSVersion + raiseMinimumTo: TLSVersion + + +def _get_cert_options_version_kwargs( + min_version: TLSVersion | None, max_version: TLSVersion | None +) -> _CertificateOptionsVersionKwargs: + """Get TLS version kwargs for + :class:`~twisted.internet.ssl.CertificateOptions` for the given limits.""" + result: _CertificateOptionsVersionKwargs = {} + if max_version: + if TWISTED_TLS_LIMITS_OFFBY1: + # lowerMaximumSecurityTo is treated as 1 version lower than the passed one + versions = list(TLSVersion.iterconstants()) + max_index = versions.index(max_version) + if max_index + 1 >= len(versions): + raise ValueError( + f"Due to an error in Twisted < 26.4.0 cannot set the maximum TLS version to {max_version.name}" + ) + max_version = versions[max_index + 1] + result["lowerMaximumSecurityTo"] = max_version + if min_version: + # We cannot pass both insecurelyLowerMinimumTo and raiseMinimumTo, + # so we need to know the direction. + + # 1.0 in Twisted 22.8.0 and older, 1.2 in Twisted 22.10.0 and newer + default_min = CertificateOptions._defaultMinimumTLSVersion + if min_version < default_min: + result["insecurelyLowerMinimumTo"] = min_version + elif min_version > default_min: + result["raiseMinimumTo"] = min_version + return result diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 3e4dae5c8..977d5a42a 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -11,7 +11,7 @@ if TYPE_CHECKING: from os import PathLike -def render_templatefile(path: str | PathLike, **kwargs: Any) -> None: +def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None: path_obj = Path(path) raw = path_obj.read_text("utf8") diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 75f357d50..b4e20c3c6 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,20 +7,14 @@ from __future__ import annotations import asyncio import os import warnings -from ftplib import FTP from importlib import import_module from pathlib import Path -from posixpath import split from typing import TYPE_CHECKING, Any, TypeVar, cast -from unittest import mock -from twisted.trial.unittest import SkipTest from twisted.web.client import Agent from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBase from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.boto import is_botocore_available -from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed from scrapy.utils.spider import DefaultSpider @@ -37,80 +31,6 @@ if TYPE_CHECKING: _T = TypeVar("_T") -def assert_gcs_environ() -> None: # pragma: no cover - warnings.warn( - "The assert_gcs_environ() function is deprecated and will be removed in a future version of Scrapy." - " Check GCS_PROJECT_ID directly.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if "GCS_PROJECT_ID" not in os.environ: - raise SkipTest("GCS_PROJECT_ID not found") - - -def skip_if_no_boto() -> None: # pragma: no cover - warnings.warn( - "The skip_if_no_boto() function is deprecated and will be removed in a future version of Scrapy." - " Check scrapy.utils.boto.is_botocore_available() directly.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if not is_botocore_available(): - raise SkipTest("missing botocore library") - - -def get_gcs_content_and_delete( - bucket: Any, path: str -) -> tuple[bytes, list[dict[str, str]], Any]: # pragma: no cover - from google.cloud import storage # noqa: PLC0415 - - warnings.warn( - "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) - bucket = client.get_bucket(bucket) - blob = bucket.get_blob(path) - content = blob.download_as_string() - acl = list(blob.acl) # loads acl before it will be deleted - bucket.delete_blob(path) - return content, acl, blob - - -def get_ftp_content_and_delete( - path: str, - host: str, - port: int, - username: str, - password: str, - use_active_mode: bool = False, -) -> bytes: # pragma: no cover - warnings.warn( - "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - ftp = FTP() - ftp.connect(host, port) - ftp.login(username, password) - if use_active_mode: - ftp.set_pasv(False) - ftp_data: list[bytes] = [] - - def buffer_data(data: bytes) -> None: - ftp_data.append(data) - - ftp.retrbinary(f"RETR {path}", buffer_data) - dirname, filename = split(path) - ftp.cwd(dirname) - ftp.delete(filename) - return b"".join(ftp_data) - - -TestSpider = create_deprecated_class("TestSpider", DefaultSpider) - - def get_reactor_settings() -> dict[str, Any]: """Return a settings dict that works with the installed reactor. @@ -128,7 +48,7 @@ def get_reactor_settings() -> dict[str, Any]: # running some 3rd-party library tests without initializing a reactor # properly. The first two cases are fine, but we cannot distinguish the # last one from them. - settings["TWISTED_ENABLED"] = False + settings["TWISTED_REACTOR_ENABLED"] = False settings["DOWNLOAD_HANDLERS"] = { "ftp": None, "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", @@ -185,29 +105,6 @@ def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: return getter -def mock_google_cloud_storage() -> tuple[Any, Any, Any]: # pragma: no cover - """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob - classes and set their proper return values. - """ - from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415 - - warnings.warn( - "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - client_mock = mock.create_autospec(Client) - - bucket_mock = mock.create_autospec(Bucket) - client_mock.get_bucket.return_value = bucket_mock - - blob_mock = mock.create_autospec(Blob) - bucket_mock.blob.return_value = blob_mock - - return (client_mock, bucket_mock, blob_mock) - - def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: # pragma: no cover warnings.warn( "The get_web_client_agent_req() function is deprecated" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py deleted file mode 100644 index 2f1acffd7..000000000 --- a/scrapy/utils/testproc.py +++ /dev/null @@ -1,77 +0,0 @@ -# pragma: no file cover -from __future__ import annotations - -import os -import sys -import warnings -from typing import TYPE_CHECKING, cast - -from twisted.internet.defer import Deferred -from twisted.internet.protocol import ProcessProtocol - -from scrapy.exceptions import ScrapyDeprecationWarning - -if TYPE_CHECKING: - from collections.abc import Iterable - - from twisted.internet.error import ProcessTerminated - from twisted.python.failure import Failure - - -warnings.warn( - "The scrapy.utils.testproc module is deprecated.", - ScrapyDeprecationWarning, -) - - -class ProcessTest: - command: str | None = None - prefix = [sys.executable, "-m", "scrapy.cmdline"] - cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 - - def execute( - self, - args: Iterable[str], - check_code: bool = True, - settings: str | None = None, - ) -> Deferred[TestProcessProtocol]: - from twisted.internet import reactor - - env = os.environ.copy() - if settings is not None: - env["SCRAPY_SETTINGS_MODULE"] = settings - assert self.command - cmd = [*self.prefix, self.command, *args] - pp = TestProcessProtocol() - pp.deferred.addCallback(self._process_finished, cmd, check_code) - reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) - return pp.deferred - - def _process_finished( - self, pp: TestProcessProtocol, cmd: list[str], check_code: bool - ) -> tuple[int, bytes, bytes]: - if pp.exitcode and check_code: - msg = f"process {cmd} exit with code {pp.exitcode}" - msg += f"\n>>> stdout <<<\n{pp.out.decode()}" - msg += "\n" - msg += f"\n>>> stderr <<<\n{pp.err.decode()}" - raise RuntimeError(msg) - return cast("int", pp.exitcode), pp.out, pp.err - - -class TestProcessProtocol(ProcessProtocol): - def __init__(self) -> None: - self.deferred: Deferred[TestProcessProtocol] = Deferred() - self.out: bytes = b"" - self.err: bytes = b"" - self.exitcode: int | None = None - - def outReceived(self, data: bytes) -> None: - self.out += data - - def errReceived(self, data: bytes) -> None: - self.err += data - - def processEnded(self, status: Failure) -> None: - self.exitcode = cast("ProcessTerminated", status.value).exitCode - self.deferred.callback(self) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py deleted file mode 100644 index e57eb802b..000000000 --- a/scrapy/utils/testsite.py +++ /dev/null @@ -1,64 +0,0 @@ -# pragma: no file cover -import warnings -from urllib.parse import urljoin - -from twisted.web import resource, server, static, util - -from scrapy.exceptions import ScrapyDeprecationWarning - -warnings.warn( - "The scrapy.utils.testsite module is deprecated.", - ScrapyDeprecationWarning, -) - - -class SiteTest: - def setUp(self): - from twisted.internet import reactor - - super().setUp() - self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - self.baseurl = f"http://localhost:{self.site.getHost().port}/" - - def tearDown(self): - super().tearDown() - self.site.stopListening() - - def url(self, path: str) -> str: - return urljoin(self.baseurl, path) - - -class NoMetaRefreshRedirect(util.Redirect): - def render(self, request: server.Request) -> bytes: - content = util.Redirect.render(self, request) - return content.replace( - b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' - ) - - -def test_site(): - r = resource.Resource() - r.putChild(b"text", static.Data(b"Works", "text/plain")) - r.putChild( - b"html", - static.Data( - b"

Works

World

", - "text/html", - ), - ) - r.putChild( - b"enc-gb18030", - static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), - ) - r.putChild(b"redirect", util.Redirect(b"/redirected")) - r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) - r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) - return server.Site(r) - - -if __name__ == "__main__": - from twisted.internet import reactor # pylint: disable=ungrouped-imports - - port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - print(f"http://localhost:{port.getHost().port}/") - reactor.run() diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index b04214c51..87df10a02 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -7,13 +7,19 @@ subclass from object_ref (instead of object). About performance: This library has a minimal performance impact when enabled, and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). + +.. note:: PyPy uses a tracing garbage collector, so objects may + remain in the ``live_refs`` longer than expected, even after they + go out of scope. If deterministic behavior is required, you may need + to explicitly trigger garbage collection or call ``trackref.live_refs.clear()``. """ from __future__ import annotations from collections import defaultdict from operator import itemgetter -from time import time +from time import monotonic_ns +from types import NoneType from typing import TYPE_CHECKING, Any from weakref import WeakKeyDictionary @@ -24,8 +30,9 @@ if TYPE_CHECKING: from typing_extensions import Self -NoneType = type(None) -live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) +live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict( + WeakKeyDictionary +) class object_ref: @@ -35,7 +42,7 @@ class object_ref: def __new__(cls, *args: Any, **kwargs: Any) -> Self: obj = object.__new__(cls) - live_refs[cls][obj] = time() + live_refs[cls][obj] = monotonic_ns() return obj @@ -43,14 +50,14 @@ class object_ref: def format_live_refs(ignore: Any = NoneType) -> str: """Return a tabular representation of tracked objects""" s = "Live References\n\n" - now = time() + now_ns = monotonic_ns() for cls, wdict in sorted(live_refs.items(), key=lambda x: x[0].__name__): if not wdict: continue if issubclass(cls, ignore): continue - oldest = min(wdict.values()) - s += f"{cls.__name__:<30} {len(wdict):6} oldest: {int(now - oldest)}s ago\n" + oldest_ns = min(wdict.values()) + s += f"{cls.__name__:<30} {len(wdict):6} oldest: {int((now_ns - oldest_ns) // 1e9)}s ago\n" return s @@ -74,4 +81,4 @@ def iter_all(class_name: str) -> Iterable[Any]: for cls, wdict in live_refs.items(): if cls.__name__ == class_name: return wdict.keys() - return [] + return () diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 097809cac..4d2bbdda2 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,31 +6,10 @@ library. from __future__ import annotations import re -import warnings -from importlib import import_module -from typing import TYPE_CHECKING, Any, TypeAlias -from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse -from warnings import warn - -from w3lib.url import __all__ as _public_w3lib_objects -from w3lib.url import add_or_replace_parameter as _add_or_replace_parameter -from w3lib.url import any_to_uri as _any_to_uri -from w3lib.url import parse_url as _parse_url - -from scrapy.exceptions import ScrapyDeprecationWarning - - -def __getattr__(name: str) -> Any: - if name in ("_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects): - obj_type = "attribute" if name == "_safe_chars" else "function" - warnings.warn( - f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.", - ScrapyDeprecationWarning, - ) - return getattr(import_module("w3lib.url"), name) - - raise AttributeError +from typing import TYPE_CHECKING, TypeAlias +from urllib.parse import ParseResult, urlparse, urlunparse +from w3lib.url import any_to_uri, parse_url if TYPE_CHECKING: from collections.abc import Iterable @@ -42,59 +21,29 @@ UrlT: TypeAlias = str | bytes | ParseResult def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: """Return True if the url belongs to any of the given domains""" - host = _parse_url(url).netloc.lower() + host = parse_url(url).netloc.lower() if not host: return False - domains = [d.lower() for d in domains] - return any((host == d) or (host.endswith(f".{d}")) for d in domains) + return any((host == d) or (host.endswith(f".{d}")) for d in map(str.lower, domains)) + + +def _spider_domains(spider: type[Spider]) -> Iterable[str]: + yield spider.name + if allowed_domains := getattr(spider, "allowed_domains", None): + yield from allowed_domains def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: """Return True if the url belongs to the given spider""" - return url_is_from_any_domain( - url, [spider.name, *getattr(spider, "allowed_domains", [])] - ) + return url_is_from_any_domain(url, _spider_domains(spider)) def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: """Return True if the url ends with one of the extensions provided""" - lowercase_path = _parse_url(url).path.lower() + lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) -def escape_ajax(url: str) -> str: - """ - Return the crawlable url - - >>> escape_ajax("www.example.com/ajax.html#!key=value") - 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue' - >>> escape_ajax("www.example.com/ajax.html?k1=v1&k2=v2#!key=value") - 'www.example.com/ajax.html?k1=v1&k2=v2&_escaped_fragment_=key%3Dvalue' - >>> escape_ajax("www.example.com/ajax.html?#!key=value") - 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue' - >>> escape_ajax("www.example.com/ajax.html#!") - 'www.example.com/ajax.html?_escaped_fragment_=' - - URLs that are not "AJAX crawlable" (according to Google) returned as-is: - - >>> escape_ajax("www.example.com/ajax.html#key=value") - 'www.example.com/ajax.html#key=value' - >>> escape_ajax("www.example.com/ajax.html#") - 'www.example.com/ajax.html#' - >>> escape_ajax("www.example.com/ajax.html") - 'www.example.com/ajax.html' - """ - warn( - "escape_ajax() is deprecated and will be removed in a future Scrapy version.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - defrag, frag = urldefrag(url) - if not frag.startswith("!"): - return url - return _add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:]) - - def add_http_if_no_scheme(url: str) -> str: """Add http as the default scheme if it is missing from the url.""" match = re.match(r"^\w+://", url, flags=re.IGNORECASE) @@ -152,7 +101,7 @@ def guess_scheme(url: str) -> str: """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" if _is_filesystem_path(url): - return _any_to_uri(url) + return any_to_uri(url) return add_http_if_no_scheme(url) @@ -184,13 +133,14 @@ def strip_url( strip_default_port and parsed_url.port and (parsed_url.scheme, parsed_url.port) - in ( + in { ("http", 80), ("https", 443), ("ftp", 21), - ) + } ): - netloc = netloc.replace(f":{parsed_url.port}", "") + port_suffix = f":{parsed_url.port}" + netloc = netloc.removesuffix(port_suffix) return urlunparse( ( diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 6d3572cb1..6bd96a215 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -3,11 +3,9 @@ from __future__ import annotations import platform import sys from importlib.metadata import version -from warnings import warn import lxml.etree -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings.default_settings import LOG_VERSIONS from scrapy.utils.ssl import get_openssl_version @@ -32,15 +30,3 @@ def get_versions( ) -> list[tuple[str, str]]: software = software or _DEFAULT_SOFTWARE return [(item, _version(item)) for item in software] - - -def scrapy_components_versions() -> list[tuple[str, str]]: # pragma: no cover - warn( - ( - "scrapy.utils.versions.scrapy_components_versions() is deprecated, " - "use scrapy.utils.versions.get_versions() instead." - ), - ScrapyDeprecationWarning, - stacklevel=2, - ) - return get_versions() diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py index 5f75d5e17..a6520ca22 100644 --- a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py @@ -28,7 +28,7 @@ if __name__ == "__main__": process = AsyncCrawlerProcess( settings={ "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", } ) process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py index 07e2d3684..55d2ef711 100644 --- a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py @@ -15,7 +15,7 @@ if __name__ == "__main__": process = AsyncCrawlerProcess( settings={ "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", } ) process.crawl(CachingHostnameResolverSpider) diff --git a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py new file mode 100644 index 000000000..f49741441 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactorless import is_reactorless + +if TYPE_CHECKING: + from asyncio import Task + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + custom_settings = { + "TWISTED_REACTOR_ENABLED": False, + } + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +def log_task_exception(task: Task[None]) -> None: + try: + task.result() + except Exception: + logging.exception("Crawl task failed") # noqa: LOG015 + + +process = AsyncCrawlerProcess() +task = process.crawl(NoRequestsSpider) +task.add_done_callback(log_task_exception) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_datauri.py b/tests/AsyncCrawlerProcess/reactorless_datauri.py index 7906cc9de..5270b7b83 100644 --- a/tests/AsyncCrawlerProcess/reactorless_datauri.py +++ b/tests/AsyncCrawlerProcess/reactorless_datauri.py @@ -12,16 +12,7 @@ class DataSpider(Spider): return {"data": response.text} -process = AsyncCrawlerProcess( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } -) +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) process.crawl(DataSpider) process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_import_hook.py b/tests/AsyncCrawlerProcess/reactorless_import_hook.py index 2f949cfc1..57e2a5c26 100644 --- a/tests/AsyncCrawlerProcess/reactorless_import_hook.py +++ b/tests/AsyncCrawlerProcess/reactorless_import_hook.py @@ -12,16 +12,7 @@ class NoRequestsSpider(scrapy.Spider): yield -process = AsyncCrawlerProcess( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } -) +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_reactor.py b/tests/AsyncCrawlerProcess/reactorless_reactor.py index a32beee22..481112ca7 100644 --- a/tests/AsyncCrawlerProcess/reactorless_reactor.py +++ b/tests/AsyncCrawlerProcess/reactorless_reactor.py @@ -3,13 +3,4 @@ from scrapy.utils.reactor import install_reactor install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") -AsyncCrawlerProcess( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } -) +AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) diff --git a/tests/AsyncCrawlerProcess/reactorless_simple.py b/tests/AsyncCrawlerProcess/reactorless_simple.py index dbe9c73b4..33b4e8cb2 100644 --- a/tests/AsyncCrawlerProcess/reactorless_simple.py +++ b/tests/AsyncCrawlerProcess/reactorless_simple.py @@ -12,16 +12,7 @@ class NoRequestsSpider(scrapy.Spider): yield -process = AsyncCrawlerProcess( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } -) +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_sleeping.py b/tests/AsyncCrawlerProcess/reactorless_sleeping.py new file mode 100644 index 000000000..12101d221 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_sleeping.py @@ -0,0 +1,20 @@ +import asyncio +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + await asyncio.sleep(int(sys.argv[1])) + + +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) + +process.crawl(SleepingSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_default.py b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_default.py deleted file mode 100644 index 1a4bc5148..000000000 --- a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_default.py +++ /dev/null @@ -1,25 +0,0 @@ -import scrapy -from scrapy.crawler import AsyncCrawlerProcess - - -class NoRequestsSpider(scrapy.Spider): - name = "no_request" - - async def start(self): - return - yield - - -process = AsyncCrawlerProcess( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } -) - -process.crawl(NoRequestsSpider) -process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py index 1814071ee..e59a365bc 100644 --- a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py +++ b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py @@ -12,12 +12,7 @@ class NoRequestsSpider(scrapy.Spider): process = AsyncCrawlerProcess( settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, + "TWISTED_REACTOR_ENABLED": False, "TELNETCONSOLE_ENABLED": False, } ) diff --git a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py index 0026a3f45..34daf8779 100644 --- a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py +++ b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py @@ -12,12 +12,7 @@ class NoRequestsSpider(scrapy.Spider): process = AsyncCrawlerProcess( settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, + "TWISTED_REACTOR_ENABLED": False, "TELNETCONSOLE_ENABLED": True, } ) diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py index f6fead718..b85f60f87 100644 --- a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py @@ -17,7 +17,7 @@ class AsyncioReactorSpider(scrapy.Spider): } -def log_task_exception(task: Task) -> None: +def log_task_exception(task: Task[None]) -> None: try: task.result() except Exception: diff --git a/tests/AsyncCrawlerRunner/no_reactor.py b/tests/AsyncCrawlerRunner/no_reactor.py new file mode 100644 index 000000000..6a473e5e8 --- /dev/null +++ b/tests/AsyncCrawlerRunner/no_reactor.py @@ -0,0 +1,22 @@ +import asyncio + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +async def main() -> None: + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +asyncio.run(main()) diff --git a/tests/AsyncCrawlerRunner/reactorless_custom_settings.py b/tests/AsyncCrawlerRunner/reactorless_custom_settings.py new file mode 100644 index 000000000..9b5efc3f9 --- /dev/null +++ b/tests/AsyncCrawlerRunner/reactorless_custom_settings.py @@ -0,0 +1,27 @@ +import asyncio + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactorless import is_reactorless + + +class NoRequestsSpider(Spider): + name = "no_request" + custom_settings = { + "TWISTED_REACTOR_ENABLED": False, + } + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +async def main() -> None: + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +asyncio.run(main()) diff --git a/tests/AsyncCrawlerRunner/reactorless_datauri.py b/tests/AsyncCrawlerRunner/reactorless_datauri.py index cd764b8db..d964f9c1c 100644 --- a/tests/AsyncCrawlerRunner/reactorless_datauri.py +++ b/tests/AsyncCrawlerRunner/reactorless_datauri.py @@ -15,18 +15,9 @@ class DataSpider(Spider): return {"data": response.text} -async def main(): +async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } - ) + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) await runner.crawl(DataSpider) diff --git a/tests/AsyncCrawlerRunner/reactorless_reactor.py b/tests/AsyncCrawlerRunner/reactorless_reactor.py index 8266ffbc0..4889b2ddc 100644 --- a/tests/AsyncCrawlerRunner/reactorless_reactor.py +++ b/tests/AsyncCrawlerRunner/reactorless_reactor.py @@ -14,18 +14,9 @@ class NoRequestsSpider(Spider): yield -async def main(): +async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } - ) + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) await runner.crawl(NoRequestsSpider) diff --git a/tests/AsyncCrawlerRunner/reactorless_simple.py b/tests/AsyncCrawlerRunner/reactorless_simple.py index 7a9e061e9..e91c7c89b 100644 --- a/tests/AsyncCrawlerRunner/reactorless_simple.py +++ b/tests/AsyncCrawlerRunner/reactorless_simple.py @@ -15,18 +15,9 @@ class NoRequestsSpider(Spider): yield -async def main(): +async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } - ) + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) await runner.crawl(NoRequestsSpider) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index 578e0029d..a2e63a0d0 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerProcess if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncio.set_event_loop(Loop()) -asyncioreactor.install() +loop = Loop() +asyncio.set_event_loop(loop) +asyncioreactor.install(loop) class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index 53d427061..7f687e7d9 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -28,7 +28,7 @@ if __name__ == "__main__": process = CrawlerProcess( settings={ "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", } ) process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py index 5cca94bed..da9c16cb8 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py +++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py @@ -15,7 +15,7 @@ if __name__ == "__main__": process = CrawlerProcess( settings={ "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", } ) process.crawl(CachingHostnameResolverSpider) diff --git a/tests/CrawlerProcess/reactorless.py b/tests/CrawlerProcess/reactorless.py index 6cfafe605..c9384010e 100644 --- a/tests/CrawlerProcess/reactorless.py +++ b/tests/CrawlerProcess/reactorless.py @@ -1,12 +1,3 @@ from scrapy.crawler import CrawlerProcess -CrawlerProcess( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } -) +CrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 9811bf90c..fae206610 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,5 +1,9 @@ # ruff: noqa: E402 +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.utils.reactor import install_reactor from tests.mockserver.dns import MockDNSServer from tests.mockserver.http import MockServer @@ -17,12 +21,14 @@ from scrapy.crawler import CrawlerRunner from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import configure_logging +if TYPE_CHECKING: + from twisted.names.common import ResolverBase + # https://stackoverflow.com/a/32784190 -def createResolver(servers=None, resolvconf=None, hosts=None): - if hosts is None: - hosts = b"/etc/hosts" if platform.getType() == "posix" else r"c:\windows\hosts" - theResolver = Resolver(resolvconf, servers) +def createResolver(servers: list[tuple[str, int]]) -> ResolverBase: + hosts = b"/etc/hosts" if platform.getType() == "posix" else r"c:\windows\hosts" + theResolver = Resolver(None, servers) hostResolver = hostsModule.Resolver(hosts) chain = [hostResolver, cache.CacheResolver(), theResolver] return resolve.ResolverChain(chain) diff --git a/tests/CrawlerRunner/no_reactor.py b/tests/CrawlerRunner/no_reactor.py new file mode 100644 index 000000000..1405de00f --- /dev/null +++ b/tests/CrawlerRunner/no_reactor.py @@ -0,0 +1,19 @@ +from twisted.python import log + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +configure_logging() +runner = CrawlerRunner() +d = runner.crawl(NoRequestsSpider) +d.addErrback(log.err) diff --git a/tests/CrawlerRunner/reactorless.py b/tests/CrawlerRunner/reactorless.py index be4eb10fb..bad2ae8d9 100644 --- a/tests/CrawlerRunner/reactorless.py +++ b/tests/CrawlerRunner/reactorless.py @@ -1,12 +1,3 @@ from scrapy.crawler import CrawlerRunner -CrawlerRunner( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } -) +CrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) diff --git a/tests/__init__.py b/tests/__init__.py index ccfabb0da..cd52ade58 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,9 +8,6 @@ import os import socket from pathlib import Path -from twisted import version as TWISTED_VERSION -from twisted.python.versions import Version - # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server # (e.g. because urllib does not fully understand the proxy spec) @@ -33,6 +30,3 @@ except socket.gaierror: def get_testdata(*paths: str) -> bytes: """Return test data""" return Path(tests_datadir, *paths).read_bytes() - - -TWISTED_KEEPS_TRACEBACKS = TWISTED_VERSION >= Version("twisted", 24, 10, 0) diff --git a/tests/ignores.txt b/tests/ignores.txt index 222288841..3717bbc95 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -1,3 +1,3 @@ -scrapy/downloadermiddlewares/cookies.py +scrapy/core/downloader/handlers/http.py scrapy/extensions/statsmailer.py -scrapy/extensions/memusage.py +scrapy/mail.py diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index e358eaca4..98fe58860 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -4,7 +4,7 @@ from collections import defaultdict from typing import Any -class DummyDB(dict): +class DummyDB(dict): # type: ignore[type-arg] """Provide dummy DBM-like interface.""" def close(self): diff --git a/tests/mockserver/dns.py b/tests/mockserver/dns.py index 7c08f237c..e19a2e61a 100644 --- a/tests/mockserver/dns.py +++ b/tests/mockserver/dns.py @@ -35,11 +35,10 @@ class MockDNSServer: [sys.executable, "-u", "-m", "tests.mockserver.dns"], stdout=PIPE, env=get_script_run_env(), + text=True, ) self.host = "127.0.0.1" - self.port = int( - self.proc.stdout.readline().strip().decode("ascii").split(":")[1] - ) + self.port = int(self.proc.stdout.readline().strip().split(":")[1]) return self def __exit__(self, exc_type, exc_value, traceback): diff --git a/tests/mockserver/ftp.py b/tests/mockserver/ftp.py index 5505ac258..22efc966b 100644 --- a/tests/mockserver/ftp.py +++ b/tests/mockserver/ftp.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import sys from argparse import ArgumentParser from pathlib import Path @@ -15,20 +16,36 @@ from tests.utils import get_script_run_env class MockFTPServer: - """Creates an FTP server on port 2121 with a default passwordless user + """Creates an FTP server on a random port with a default passwordless user (anonymous) and a temporary root path that you can read from the :attr:`path` attribute.""" + def __init__(self) -> None: + self.proc: Popen[str] | None = None + self.host: str = "127.0.0.1" + self.port: int | None = None + self.path: Path | None = None + def __enter__(self): self.path = Path(mkdtemp()) self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)], stderr=PIPE, env=get_script_run_env(), + text=True, ) for line in self.proc.stderr: - if b"starting FTP server" in line: + if "starting FTP server" in line and ( + m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line) + ): + self.port = int(m.group(2)) break + else: + self.proc.kill() + self.proc.communicate() + raise RuntimeError( + "The FTP server failed to start or the output is unrecognized" + ) return self def __exit__(self, exc_type, exc_value, traceback): @@ -37,12 +54,12 @@ class MockFTPServer: self.proc.communicate() def url(self, path): - return "ftp://127.0.0.1:2121/" + path + return f"ftp://{self.host}:{self.port}/{path}" def main() -> None: parser = ArgumentParser() - parser.add_argument("-d", "--directory") + parser.add_argument("-d", "--directory", required=True) args = parser.parse_args() authorizer = DummyAuthorizer() @@ -50,7 +67,7 @@ def main() -> None: authorizer.add_anonymous(args.directory, perm=full_permissions) handler = FTPHandler handler.authorizer = authorizer - address = ("127.0.0.1", 2121) + address = ("127.0.0.1", 0) server = FTPServer(address, handler) server.serve_forever() diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index d7c892828..7ad873c02 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -14,6 +14,7 @@ from .http_resources import ( BrokenChunkedResource, BrokenDownloadResource, ChunkedResource, + ClientIPResource, Compress, ContentLengthHeaderResource, Delay, @@ -33,6 +34,7 @@ from .http_resources import ( ResponseHeadersResource, SetCookie, Status, + UriResource, ) @@ -72,6 +74,7 @@ class Root(resource.Resource): self.putChild(b"wait", ForeverTakingResource()) self.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) self.putChild(b"host", HostHeaderResource()) + self.putChild(b"client-ip", ClientIPResource()) self.putChild(b"broken", BrokenDownloadResource()) self.putChild(b"chunked", ChunkedResource()) self.putChild(b"broken-chunked", BrokenChunkedResource()) @@ -82,8 +85,9 @@ class Root(resource.Resource): self.putChild(b"duplicate-header", DuplicateHeaderResource()) self.putChild(b"response-headers", ResponseHeadersResource()) self.putChild(b"set-cookie", SetCookie()) + self.putChild(b"uri", UriResource()) - def getChild(self, name, request): + def getChild(self, path, request): return self def render(self, request): diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py index 440d38fb9..7b38409ff 100644 --- a/tests/mockserver/http_base.py +++ b/tests/mockserver/http_base.py @@ -34,7 +34,7 @@ class BaseMockServer(ABC): if not self.listen_http and not self.listen_https: raise ValueError("At least one of listen_http and listen_https must be set") - self.proc: Popen | None = None + self.proc: Popen[str] | None = None self.host: str = "127.0.0.1" self.http_port: int | None = None self.https_port: int | None = None @@ -44,13 +44,14 @@ class BaseMockServer(ABC): [sys.executable, "-u", "-m", self.module_name, *self.get_additional_args()], stdout=PIPE, env=get_script_run_env(), + text=True, ) if self.listen_http: - http_address = self.proc.stdout.readline().strip().decode("ascii") + http_address = self.proc.stdout.readline().strip() http_parsed = urlparse(http_address) self.http_port = http_parsed.port if self.listen_https: - https_address = self.proc.stdout.readline().strip().decode("ascii") + https_address = self.proc.stdout.readline().strip() https_parsed = urlparse(https_address) self.https_port = https_parsed.port return self @@ -105,6 +106,16 @@ def main_factory( default=None, help="SSL cipher string (optional)", ) + parser.add_argument( + "--tls-min-version", + default=None, + help="Minimum accepted TLS version (optional)", + ) + parser.add_argument( + "--tls-max-version", + default=None, + help="Maximum accepted TLS version (optional)", + ) args = parser.parse_args() context_factory_kw = {} if args.keyfile: @@ -113,6 +124,10 @@ def main_factory( context_factory_kw["certfile"] = args.certfile if args.cipher_string: context_factory_kw["cipher_string"] = args.cipher_string + if args.tls_min_version: + context_factory_kw["tls_min_version"] = args.tls_min_version + if args.tls_max_version: + context_factory_kw["tls_max_version"] = args.tls_max_version context_factory = ssl_context_factory(**context_factory_kw) https_port = reactor.listenSSL(0, factory, context_factory) diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py index 91577aa6b..98ac6cf6a 100644 --- a/tests/mockserver/http_resources.py +++ b/tests/mockserver/http_resources.py @@ -3,6 +3,7 @@ from __future__ import annotations import gzip import json import random +from typing import TYPE_CHECKING, ParamSpec, TypeVar from urllib.parse import urlencode from twisted.internet.task import deferLater @@ -12,6 +13,16 @@ from twisted.web.util import Redirect, redirectTo from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from collections.abc import Callable + + from twisted.internet.defer import Deferred + from twisted.web.http import Request + + +_T = TypeVar("_T") +_P = ParamSpec("_P") + def getarg(request, name, default=None, type_=None): if name in request.args: @@ -56,6 +67,18 @@ class HostHeaderResource(resource.Resource): return request.requestHeaders.getRawHeaders(b"host")[0] +class ClientIPResource(resource.Resource): + """ + A testing resource which renders itself as the request client IP address. + """ + + def render(self, request): + client_address = request.getClientAddress() + if client_address is None or client_address.host is None: + return b"" + return to_bytes(client_address.host) + + class PayloadResource(resource.Resource): """ A testing resource which renders itself as the contents of the request body @@ -74,7 +97,14 @@ class PayloadResource(resource.Resource): class LeafResource(resource.Resource): isLeaf = True - def deferRequest(self, request, delay, f, *a, **kw): + def deferRequest( + self, + request: Request, + delay: float, + f: Callable[_P, _T], + *a: _P.args, + **kw: _P.kwargs, + ) -> Deferred[_T]: from twisted.internet import reactor def _cancelrequest(_): @@ -209,7 +239,7 @@ class ArbitraryLengthPayloadResource(LeafResource): class NoMetaRefreshRedirect(Redirect): def render(self, request: server.Request) -> bytes: - content = Redirect.render(self, request) + content: bytes = Redirect.render(self, request) return content.replace( b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' ) @@ -318,7 +348,7 @@ class ResponseHeadersResource(resource.Resource): def render(self, request): body = json.loads(request.content.read().decode()) for header_name, header_value in body.items(): - request.responseHeaders.addRawHeader(header_name, header_value) + request.responseHeaders.setRawHeaders(header_name, [header_value]) return json.dumps(body).encode("utf-8") diff --git a/tests/mockserver/mitm_proxy.py b/tests/mockserver/mitm_proxy.py new file mode 100644 index 000000000..56620f84e --- /dev/null +++ b/tests/mockserver/mitm_proxy.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import re +import sys +from pathlib import Path +from subprocess import PIPE, Popen +from urllib.parse import urlsplit, urlunsplit + + +class MitmProxy: + auth_user = "scrapy" + auth_pass = "scrapy" + + def __init__(self, mode: str | None = None) -> None: + self.mode = mode + + def start(self) -> str: + script = """ +import sys +from mitmproxy.tools.main import mitmdump +sys.argv[0] = "mitmdump" +sys.exit(mitmdump()) + """ + cert_path = Path(__file__).parent.parent.resolve() / "keys" + args = [ + "--listen-host", + "127.0.0.1", + "--listen-port", + "0", + "--proxyauth", + f"{self.auth_user}:{self.auth_pass}", + "--set", + f"confdir={cert_path}", + "--ssl-insecure", + "-s", + str(Path(__file__).with_name("mitm_proxy_addon.py")), + ] + if self.mode: + args += ["--mode", self.mode] + self.proc: Popen[str] = Popen( + [ + sys.executable, + "-u", + "-c", + script, + *args, + ], + stdout=PIPE, + text=True, + ) + assert self.proc.stdout is not None + scheme = "socks5" if self.mode == "socks5" else "http" + line = "" + for line in self.proc.stdout: + m = re.search(r"listening at (?:\w+://)?([^:]+:\d+)", line) + if m: + host_port = m.group(1) + return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host_port}" + self.stop() + raise RuntimeError(f"Failed to parse mitmdump output: {line}") + + def stop(self) -> None: + self.proc.kill() + self.proc.communicate() + + +def wrong_credentials(proxy_url: str) -> str: + bad_auth_proxy = list(urlsplit(proxy_url)) + bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@") + return urlunsplit(bad_auth_proxy) diff --git a/tests/mockserver/mitm_proxy_addon.py b/tests/mockserver/mitm_proxy_addon.py new file mode 100644 index 000000000..04ba2e4a1 --- /dev/null +++ b/tests/mockserver/mitm_proxy_addon.py @@ -0,0 +1,5 @@ +def response(flow) -> None: + # add custom headers to be able to check that the request went through the proxy + flow.response.headers["X-Via-Mitmproxy"] = "1" + if flow.client_conn.tls_established: + flow.response.headers["X-Via-Mitmproxy-TLS"] = "1" diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py index 5f23dd2c4..fdea666e1 100644 --- a/tests/mockserver/simple_https.py +++ b/tests/mockserver/simple_https.py @@ -13,7 +13,7 @@ class Root(resource.Resource): resource.Resource.__init__(self) self.putChild(b"file", Data(b"0123456789", "text/plain")) - def getChild(self, name, request): + def getChild(self, path, request): return self @@ -21,11 +21,21 @@ class SimpleMockServer(BaseMockServer): listen_http = False module_name = "tests.mockserver.simple_https" - def __init__(self, keyfile: str, certfile: str, cipher_string: str | None): + def __init__( + self, + keyfile: str, + certfile: str, + *, + cipher_string: str | None = None, + tls_min_version: str | None = None, + tls_max_version: str | None = None, + ): super().__init__() self.keyfile = keyfile self.certfile = certfile - self.cipher_string = cipher_string or "" + self.cipher_string = cipher_string + self.tls_min_version = tls_min_version + self.tls_max_version = tls_max_version def get_additional_args(self) -> list[str]: args = [ @@ -36,6 +46,10 @@ class SimpleMockServer(BaseMockServer): ] if self.cipher_string is not None: args.extend(["--cipher-string", self.cipher_string]) + if self.tls_min_version is not None: + args.extend(["--tls-min-version", self.tls_min_version]) + if self.tls_max_version is not None: + args.extend(["--tls-max-version", self.tls_max_version]) return args diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py index f43152f92..5c4ca7457 100644 --- a/tests/mockserver/utils.py +++ b/tests/mockserver/utils.py @@ -1,23 +1,53 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING, cast +from cryptography.hazmat.primitives.serialization import load_pem_private_key +from cryptography.x509 import load_pem_x509_certificate from OpenSSL import SSL -from twisted.internet import ssl +from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey +from twisted.internet.ssl import CertificateOptions, ContextFactory +from scrapy.core.downloader.tls import _TWISTED_VERSION_MAP +from scrapy.utils._deps_compat import PYOPENSSL_X509_DEPRECATED from scrapy.utils.python import to_bytes +from scrapy.utils.ssl import _get_cert_options_version_kwargs + +if TYPE_CHECKING: + from twisted.internet.interfaces import IOpenSSLContextFactory def ssl_context_factory( - keyfile="keys/localhost.key", certfile="keys/localhost.crt", cipher_string=None -): - factory = ssl.DefaultOpenSSLContextFactory( - str(Path(__file__).parent.parent / keyfile), - str(Path(__file__).parent.parent / certfile), + keyfile: str = "keys/localhost.key", + certfile: str = "keys/localhost.crt", + *, + cipher_string: str | None = None, + tls_min_version: str | None = None, + tls_max_version: str | None = None, +) -> IOpenSSLContextFactory: + keyfile_path = Path(__file__).parent.parent / keyfile + certfile_path = Path(__file__).parent.parent / certfile + + if PYOPENSSL_X509_DEPRECATED: + cert = load_pem_x509_certificate(certfile_path.read_bytes()) + key = load_pem_private_key(keyfile_path.read_bytes(), password=None) + else: + cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes()) # type: ignore[assignment] + key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes()) # type: ignore[assignment] + + tls_min = _TWISTED_VERSION_MAP.get(tls_min_version) if tls_min_version else None + tls_max = _TWISTED_VERSION_MAP.get(tls_max_version) if tls_max_version else None + tls_version_kwargs = _get_cert_options_version_kwargs(tls_min, tls_max) + # https://github.com/twisted/twisted/issues/12638 + factory: CertificateOptions = CertificateOptions( + privateKey=key, # type: ignore[arg-type] + certificate=cert, # type: ignore[arg-type] + **tls_version_kwargs, ) if cipher_string: ctx = factory.getContext() # disabling TLS1.3 because it unconditionally enables some strong ciphers ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_3) ctx.set_cipher_list(to_bytes(cipher_string)) - return factory + return cast("ContextFactory", factory) diff --git a/tests/spiders.py b/tests/spiders.py index 363a09fad..612dc11c9 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio import time +from typing import TYPE_CHECKING, Any from urllib.parse import urlencode from twisted.internet import defer @@ -20,9 +21,18 @@ from scrapy.spiders.crawl import CrawlSpider, Rule from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future from scrapy.utils.test import get_from_asyncio_queue +if TYPE_CHECKING: + from tests.mockserver.http import MockServer + class MockServerSpider(Spider): - def __init__(self, *args, mockserver=None, is_secure=False, **kwargs): + def __init__( + self, + *args, + mockserver: MockServer | None = None, + is_secure: bool = False, + **kwargs: Any, + ): super().__init__(*args, **kwargs) self.mockserver = mockserver self.is_secure = is_secure @@ -31,9 +41,9 @@ class MockServerSpider(Spider): class MetaSpider(MockServerSpider): name = "meta" - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - self.meta = {} + self.meta: dict[str, Any] = {} def closed(self, reason): self.meta["close_reason"] = reason @@ -84,19 +94,19 @@ class DelaySpider(MetaSpider): class LogSpider(MetaSpider): name = "log_spider" - def log_debug(self, message: str, extra: dict | None = None): + def log_debug(self, message: str, extra: dict[str, Any] | None = None): self.logger.debug(message, extra=extra) - def log_info(self, message: str, extra: dict | None = None): + def log_info(self, message: str, extra: dict[str, Any] | None = None): self.logger.info(message, extra=extra) - def log_warning(self, message: str, extra: dict | None = None): + def log_warning(self, message: str, extra: dict[str, Any] | None = None): self.logger.warning(message, extra=extra) - def log_error(self, message: str, extra: dict | None = None): + def log_error(self, message: str, extra: dict[str, Any] | None = None): self.logger.error(message, extra=extra) - def log_critical(self, message: str, extra: dict | None = None): + def log_critical(self, message: str, extra: dict[str, Any] | None = None): self.logger.critical(message, extra=extra) def parse(self, response): @@ -212,7 +222,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider): class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): - name = "asyncdef_deferred_wrapped" + name = "asyncdef_deferred_maybe_wrapped" async def parse(self, response): await maybe_deferred_to_future(defer.succeed(None)) @@ -407,7 +417,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): """ name = "crawl_spider_with_parse_method" - custom_settings: dict = { + custom_settings: dict[str, Any] = { "RETRY_HTTP_CODES": [], # no need to retry } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) diff --git a/tests/test_addons.py b/tests/test_addons.py index 5c26e4651..14ebddda8 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -71,6 +71,37 @@ class TestAddonManager: manager = crawler.addons assert not manager.addons + def test_notconfigured_with_args(self): + class NotConfiguredAddon: + def update_settings(self, settings): + raise NotConfigured("addon disabled reason") + + settings_dict = { + "ADDONS": {NotConfiguredAddon: 0}, + } + with patch("scrapy.addons.logger") as logger_mock: + crawler = get_crawler(settings_dict=settings_dict) + assert not crawler.addons.addons + logger_mock.warning.assert_called_once_with( + "Disabled %(clspath)s: %(eargs)s", + {"clspath": NotConfiguredAddon, "eargs": "addon disabled reason"}, + extra={"crawler": crawler}, + ) + + def test_no_update_settings(self): + class PreCrawlerOnlyAddon: + @classmethod + def update_pre_crawler_settings(cls, settings): + settings.set("PRE_CRAWLER_KEY", "value", priority="addon") + + settings_dict = { + "ADDONS": {PreCrawlerOnlyAddon: 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + assert len(manager.addons) == 1 + assert isinstance(manager.addons[0], PreCrawlerOnlyAddon) + def test_load_settings_order(self): # Get three addons with different settings addonlist = [] @@ -110,7 +141,7 @@ class TestAddonManager: runner_cls = ( CrawlerRunner - if settings_dict.get("TWISTED_ENABLED", True) + if settings_dict.get("TWISTED_REACTOR_ENABLED", True) else AsyncCrawlerRunner ) @@ -130,6 +161,7 @@ class TestAddonManager: settings.set("KEY", 0, priority="default") runner = runner_cls(settings) crawler = runner.create_crawler(Spider) + crawler._apply_settings() assert crawler.settings.getint("KEY") == 20 def test_fallback_workflow(self): @@ -201,7 +233,7 @@ class TestAddonManager: settings.set("KEY", "default", priority="default") runner_cls = ( CrawlerRunner - if settings.getbool("TWISTED_ENABLED", True) + if settings.getbool("TWISTED_REACTOR_ENABLED", True) else AsyncCrawlerRunner ) runner = runner_cls(settings) diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index 11c821f8d..ef1e50c0c 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -1,14 +1,2 @@ -"""A test extension used to check the settings loading order""" - - -class TestExtension: - def __init__(self, settings): - settings.set("TEST1", f"{settings['TEST1']} + started") - - @classmethod - def from_crawler(cls, crawler): - return cls(crawler.settings) - - class DummyExtension: pass diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py index 32b15e191..ec71bba0c 100644 --- a/tests/test_cmdline/settings.py +++ b/tests/test_cmdline/settings.py @@ -1,7 +1,7 @@ from pathlib import Path EXTENSIONS = { - "tests.test_cmdline.extensions.TestExtension": 0, + "tests.test_cmdline.extensions.DummyExtension": 0, } TEST1 = "default" diff --git a/tests/test_command_check.py b/tests/test_command_check.py index dc07ed695..815f87026 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -4,7 +4,7 @@ import sys from io import StringIO from typing import TYPE_CHECKING from unittest import TestCase -from unittest.mock import Mock, PropertyMock, call, patch +from unittest.mock import MagicMock, Mock, PropertyMock, call, patch from scrapy.commands.check import Command, TextTestResult from tests.test_commands import TestProjectBase @@ -46,10 +46,17 @@ class CheckSpider(scrapy.Spider): ) def _test_contract( - self, proj_path: Path, contracts: str = "", parse_def: str = "pass" + self, + proj_path: Path, + contracts: str = "", + parse_def: str = "pass", + use_reactor: bool = True, ) -> None: self._write_contract(proj_path, contracts, parse_def) - ret, out, err = proc("check", cwd=proj_path) + args = ["check"] + if not use_reactor: + args += ["-s", "TWISTED_REACTOR_ENABLED=False"] + ret, out, err = proc(*args, cwd=proj_path) assert "F" not in out assert "OK" in err assert ret == 0 @@ -63,6 +70,15 @@ class CheckSpider(scrapy.Spider): """ self._test_contract(proj_path, contracts, parse_def) + def test_check_no_reactor(self, proj_path: Path) -> None: + contracts = """ + @returns requests 1 + """ + parse_def = """ + yield scrapy.Request(url='http://next-url.com') + """ + self._test_contract(proj_path, contracts, parse_def, use_reactor=False) + def test_check_returns_items_contract(self, proj_path: Path) -> None: contracts = """ @returns items 1 @@ -117,7 +133,7 @@ class CheckSpider(scrapy.Spider): def test_printSummary_with_unsuccessful_test_result_without_errors_and_without_failures( self, ) -> None: - result = TextTestResult(Mock(), descriptions=False, verbosity=1) + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) start_time = 1.0 stop_time = 2.0 result.testsRun = 5 @@ -131,7 +147,7 @@ class CheckSpider(scrapy.Spider): def test_printSummary_with_unsuccessful_test_result_with_only_failures( self, ) -> None: - result = TextTestResult(Mock(), descriptions=False, verbosity=1) + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) start_time = 1.0 stop_time = 2.0 result.testsRun = 5 @@ -142,7 +158,7 @@ class CheckSpider(scrapy.Spider): mock_write.assert_called_with(" (failures=1)") def test_printSummary_with_unsuccessful_test_result_with_only_errors(self) -> None: - result = TextTestResult(Mock(), descriptions=False, verbosity=1) + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) start_time = 1.0 stop_time = 2.0 result.testsRun = 5 @@ -155,7 +171,7 @@ class CheckSpider(scrapy.Spider): def test_printSummary_with_unsuccessful_test_result_with_both_failures_and_errors( self, ) -> None: - result = TextTestResult(Mock(), descriptions=False, verbosity=1) + result = TextTestResult(MagicMock(), descriptions=False, verbosity=1) start_time = 1.0 stop_time = 2.0 result.testsRun = 5 @@ -170,7 +186,9 @@ class CheckSpider(scrapy.Spider): output = StringIO() sys.stdout = output cmd = Command() - cmd.settings = Mock(getwithbase=Mock(return_value={})) + cmd.settings = Mock( + get_component_priority_dict_with_base=Mock(return_value={}), + ) cm_cls_mock.return_value = cm_mock = Mock() spider_loader_mock = Mock() cmd.crawler_process = Mock(spider_loader=spider_loader_mock) @@ -195,7 +213,9 @@ class CheckSpider(scrapy.Spider): self, cm_cls_mock ) -> None: cmd = Command() - cmd.settings = Mock(getwithbase=Mock(return_value={})) + cmd.settings = Mock( + get_component_priority_dict_with_base=Mock(return_value={}), + ) cm_cls_mock.return_value = cm_mock = Mock() spider_loader_mock = Mock() cmd.crawler_process = Mock(spider_loader=spider_loader_mock) diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 5a223d122..dd69d36ba 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -124,3 +124,22 @@ class MySpider(scrapy.Spider): not in log ) assert "Spider closed (finished)" in log + + def test_no_reactor(self, proj_path: Path) -> None: + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('It works!') + return + yield +""" + log = self.get_log( + spider_code, proj_path, args=("-s", "TWISTED_REACTOR_ENABLED=False") + ) + assert "[myspider] DEBUG: It works!" in log + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 4f4f19c3c..d98dac968 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -30,3 +30,9 @@ class TestFetchCommand: out = out.replace("\r", "") # required on win32 assert "Server: TwistedWeb" in out assert "Content-Type: text/plain" in out + + def test_no_reactor(self, mockserver: MockServer) -> None: + _, out, _ = proc( + "fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text") + ) + assert out.strip() == "Works" diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index fd9505060..67e3eb50a 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -9,7 +9,7 @@ from tests.test_commands import TestProjectBase from tests.utils.cmdline import call, proc -def find_in_file(filename: Path, regex: str) -> re.Match | None: +def find_in_file(filename: Path, regex: str) -> re.Match[str] | None: """Find first pattern occurrence in file""" pattern = re.compile(regex) with filename.open("r", encoding="utf-8") as f: diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 85529e607..c210a06e8 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -513,3 +513,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert namespace.depth == 2 assert namespace.spider == self.spider_name assert namespace.verbose + + def test_no_reactor(self, proj_path: Path, mockserver: MockServer) -> None: + _, out, stderr = proc( + "parse", + "--spider", + "asyncdef_asyncio_return", + "-c", + "parse", + mockserver.url("/html"), + "-s", + "TWISTED_REACTOR_ENABLED=False", + cwd=proj_path, + ) + assert "INFO: Got response 200" in stderr + assert "{'id': 1}" in out + assert "{'id': 2}" in out diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index b5408ccfe..b1455611e 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -213,6 +213,17 @@ class MySpider(scrapy.Spider): in log ) + def test_no_reactor(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR_ENABLED=False", + ], + ) + assert "Not using a Twisted reactor" in log + def test_output(self, tmp_path: Path) -> None: spider_code = """ import scrapy diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 1417bc623..1585835cc 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -4,7 +4,7 @@ import os import sys from io import BytesIO from pathlib import Path -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING import pytest from pexpect.popen_spawn import PopenSpawn @@ -22,6 +22,12 @@ class TestShellCommand: _, out, _ = proc("shell", "-c", "item") assert "{}" in out + def test_empty_no_reactor(self) -> None: + _, out, _ = proc( + "shell", "-c", "item", "--set", "TWISTED_REACTOR_ENABLED=False" + ) + assert "{}" in out + def test_response_body(self, mockserver: MockServer) -> None: _, out, _ = proc("shell", mockserver.url("/text"), "-c", "response.body") assert "Works" in out @@ -125,6 +131,14 @@ class TestShellCommand: assert ret == 0, err assert "RuntimeError: There is no current event loop in thread" not in err + def test_shell_fetch_no_reactor(self, mockserver: MockServer) -> None: + url = mockserver.url("/html") + code = f"fetch('{url}')" + ret, _, err = proc( + "shell", "-c", code, "--set", "TWISTED_REACTOR_ENABLED=False" + ) + assert ret == 0, err + class TestInteractiveShell: def test_fetch(self, mockserver: MockServer) -> None: @@ -137,14 +151,13 @@ class TestInteractiveShell: env = os.environ.copy() env["SCRAPY_PYTHON_SHELL"] = "python" logfile = BytesIO() - # https://github.com/python/typeshed/issues/14915 - p = PopenSpawn(args, env=cast("os._Environ", env), timeout=5) + p = PopenSpawn(args, env=env, timeout=5) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") p.sendline(f"fetch('{mockserver.url('/')}')") p.sendline("type(response)") p.expect_exact("HtmlResponse") p.sendeof() - p.wait() + p.wait() # type: ignore[no-untyped-call] logfile.seek(0) assert "Traceback" not in logfile.read().decode() diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 4eed09fcd..7ac6c4fb0 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -79,7 +79,7 @@ class TestStartprojectCommand: def get_permissions_dict( - path: str | os.PathLike, renamings=None, ignore=None + path: str | os.PathLike[str], renamings=None, ignore=None ) -> dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) @@ -257,7 +257,7 @@ class TestStartprojectTemplates: assert actual_permissions == expected_permissions - def test_startproject_permissions_umask_022(self, tmp_path: Path) -> None: + def test_startproject_permissions_umask_002(self, tmp_path: Path) -> None: """Check that generated files have the right permissions when the system uses a umask value that causes new files to have different permissions than those from the template folder.""" diff --git a/tests/test_commands.py b/tests/test_commands.py index 07bd0b45c..d7b7a9ff2 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -12,6 +12,7 @@ import pytest import scrapy from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.reactor import _asyncio_reactor_path from tests.utils.cmdline import call, proc @@ -28,6 +29,50 @@ class EmptyCommand(ScrapyCommand): pass +class TestHelpDeprecation: + def test_calling_help_is_deprecated(self) -> None: + command = EmptyCommand() + with pytest.warns( + ScrapyDeprecationWarning, + match=r"ScrapyCommand\.help\(\) is deprecated, use long_desc\(\) instead\.", + ): + result = command.help() + # help() still delegates to long_desc() for backward compatibility. + assert result == command.long_desc() + + def test_overriding_help_is_deprecated(self) -> None: + class HelpCommand(ScrapyCommand): + def short_desc(self) -> str: + return "" + + def run(self, args: list[str], opts: argparse.Namespace) -> None: + pass + + def help(self) -> str: + return "custom help" + + with pytest.warns( + ScrapyDeprecationWarning, + match=r"The ScrapyCommand\.help\(\) method is deprecated and " + r"overriding it, as the .*HelpCommand class does, has no effect; " + r"override long_desc\(\) instead\.", + ): + HelpCommand() + + def test_not_overriding_help_does_not_warn(self, recwarn) -> None: + # Commands that do not override help() must not emit the + # override-deprecation warning when instantiated, including subclasses + # several levels below ScrapyCommand (as the built-in commands are). + class SubCommand(EmptyCommand): + pass + + EmptyCommand() + SubCommand() + assert not [ + w for w in recwarn.list if issubclass(w.category, ScrapyDeprecationWarning) + ] + + class TestCommandSettings: def setup_method(self): self.command = EmptyCommand() @@ -214,9 +259,7 @@ class MySpider(scrapy.Spider): self._append_settings(proj_path / self.project_name, "TWISTED_REACTOR = None\n") self._assert_spider_works(self.NORMAL_MSG, proj_path, "sp") - self._assert_spider_asyncio_fail( - self.NORMAL_MSG, proj_path, "aiosp", "-s", "TWISTED_REACTOR=" - ) + self._assert_spider_asyncio_fail(self.NORMAL_MSG, proj_path, "aiosp") def test_spider_settings_asyncio(self, proj_path: Path) -> None: """The reactor is set via the spider settings to the asyncio value. @@ -345,14 +388,18 @@ Unknown command: abc class TestBenchCommand: - def test_run(self) -> None: - _, _, err = proc( + @pytest.mark.parametrize("use_reactor", [True, False]) + def test_run(self, use_reactor: bool) -> None: + args: list[str] = [ "bench", "-s", "LOGSTATS_INTERVAL=0.001", "-s", "CLOSESPIDER_TIMEOUT=0.01", - ) + ] + if not use_reactor: + args += ["-s", "TWISTED_REACTOR_ENABLED=False"] + _, _, err = proc(*args) assert "INFO: Crawled" in err assert "Unhandled Error" not in err assert "log_count/ERROR" not in err @@ -374,27 +421,28 @@ class TestViewCommand: class TestHelpMessage(TestProjectBase): - COMMANDS = [ - "parse", - "startproject", - "view", - "crawl", - "edit", - "list", - "fetch", - "settings", - "shell", - "runspider", - "version", - "genspider", - "check", - "bench", - ] - - def test_help_messages(self, proj_path: Path) -> None: - for command in self.COMMANDS: - _, out, _ = proc(command, "-h", cwd=proj_path) - assert "Usage" in out + @pytest.mark.parametrize( + "command", + [ + "parse", + "startproject", + "view", + "crawl", + "edit", + "list", + "fetch", + "settings", + "shell", + "runspider", + "version", + "genspider", + "check", + "bench", + ], + ) + def test_help_messages(self, proj_path: Path, command: str) -> None: + _, out, _ = proc(command, "-h", cwd=proj_path) + assert "Usage" in out class TestPopCommandName: diff --git a/tests/test_contracts.py b/tests/test_contracts.py index a35ef7010..e80945b93 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -3,7 +3,6 @@ from unittest import TextTestResult import pytest from twisted.python import failure -from scrapy import FormRequest from scrapy.contracts import Contract, ContractsManager from scrapy.contracts.default import ( CallbackKeywordArgumentsContract, @@ -34,6 +33,12 @@ class ResponseMetaMock(ResponseMock): meta = None +class TaggedRequest(Request): + def __init__(self, url, contract_tag=None, **kwargs): + super().__init__(url, **kwargs) + self.contract_tag = contract_tag + + class CustomSuccessContract(Contract): name = "custom_success_contract" @@ -49,12 +54,13 @@ class CustomFailContract(Contract): raise TypeError("Error in adjust_request_args") -class CustomFormContract(Contract): - name = "custom_form" - request_cls = FormRequest +class CustomTaggedRequestContract(Contract): + name = "custom_tagged_request" + request_cls = TaggedRequest def adjust_request_args(self, args): - args["formdata"] = {"name": "scrapy"} + args["contract_tag"] = "custom" + args["method"] = "POST" return args @@ -179,10 +185,10 @@ class DemoSpider(Spider): @returns items 1 1 """ - def custom_form(self, response): + def custom_tagged_request(self, response): """ @url http://scrapy.org - @custom_form + @custom_tagged_request """ def invalid_regex(self, response): @@ -253,7 +259,7 @@ class TestContractsManager: MetadataContract, ReturnsContract, ScrapesContract, - CustomFormContract, + CustomTaggedRequestContract, CustomSuccessContract, CustomFailContract, ] @@ -382,7 +388,7 @@ class TestContractsManager: request = self.conman.from_method(spider.returns_item_meta, self.results) assert request.meta["key"] == "example" response.meta = request.meta - request.callback(ResponseMetaMock) + request.callback(response) assert response.meta["key"] == "example" self.should_succeed() @@ -470,14 +476,14 @@ class TestContractsManager: # invalid regex request = self.conman.from_method(spider.invalid_regex, self.results) - self.should_succeed() + assert request is None # invalid regex with valid contract request = self.conman.from_method( spider.invalid_regex_with_valid_contract, self.results ) - self.should_succeed() request.callback(response) + self.should_succeed() def test_custom_contracts(self): self.conman.from_spider(CustomContractSuccessSpider(), self.results) @@ -533,17 +539,21 @@ class TestContractsManager: assert crawler.spider.visited == 2 - def test_form_contract(self): + def test_custom_tagged_request_contract(self): spider = DemoSpider() - request = self.conman.from_method(spider.custom_form, self.results) + request = self.conman.from_method(spider.custom_tagged_request, self.results) assert request.method == "POST" - assert isinstance(request, FormRequest) + assert isinstance(request, TaggedRequest) + assert request.contract_tag == "custom" def test_inherited_contracts(self): spider = InheritsDemoSpider() requests = self.conman.from_spider(spider, self.results) assert requests + assert any( + isinstance(request, TaggedRequest) for request in requests if request + ) class CustomFailContractPreProcess(Contract): @@ -568,7 +578,7 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPreProcess(spider.returns_request) - conman = ContractsManager([contract]) + conman = ContractsManager([UrlContract, ReturnsContract, contract]) request = conman.from_method(spider.returns_request, self.results) contract.add_pre_hook(request, self.results) @@ -582,7 +592,7 @@ class TestCustomContractPrePostProcess: spider = DemoSpider() response = ResponseMock() contract = CustomFailContractPostProcess(spider.returns_request) - conman = ContractsManager([contract]) + conman = ContractsManager([UrlContract, ReturnsContract, contract]) request = conman.from_method(spider.returns_request, self.results) contract.add_post_hook(request, self.results) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 2a098f9f3..e348bfb7f 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -1,23 +1,30 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, cast import OpenSSL.SSL import pytest from pytest_twisted import async_yield_fixture +from twisted.internet.protocol import Factory +from twisted.internet.protocol import Protocol as TxProtocol +from twisted.internet.ssl import optionsForClientTLS +from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse -from scrapy.core.downloader import Downloader, Slot +from scrapy.core.downloader import Downloader, Slot, tls from scrapy.core.downloader.contextfactory import ( - ScrapyClientContextFactory, - load_context_factory_from_settings, + _load_context_factory_from_settings, + _ScrapyClientContextFactory, ) from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.settings import Settings +from scrapy.utils._deps_compat import ( + PYOPENSSL_SET_CIPHER_LIST_TMP_CONN, + TWISTED_TLS_NEW_IMPL, +) from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes @@ -35,13 +42,11 @@ if TYPE_CHECKING: class TestSlot: def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) - assert repr(slot) == "Slot(concurrency=8, delay=0.10, randomize_delay=True)" + assert repr(slot) == "Slot(concurrency=8, delay=0.1, randomize_delay=True)" -@pytest.mark.requires_reactor +@pytest.mark.requires_reactor # this test is related to the Twisted HTTP code class TestContextFactoryBase: - context_factory = None - @async_yield_fixture async def server_url(self, tmp_path): (tmp_path / "file").write_bytes(b"0123456789") @@ -61,7 +66,7 @@ class TestContextFactoryBase: return reactor.listenSSL( 0, site, - contextFactory=self.context_factory or ssl_context_factory(), + contextFactory=ssl_context_factory(), interface="127.0.0.1", ) @@ -97,37 +102,87 @@ class TestContextFactoryBase: class TestContextFactory(TestContextFactoryBase): - @coroutine_test - async def testPayload(self, server_url: str) -> None: - s = "0123456789" * 10 + @pytest.fixture + def factory(self) -> _ScrapyClientContextFactory: crawler = get_crawler() - settings = Settings() - client_context_factory = load_context_factory_from_settings(settings, crawler) - body = await self.get_page( - server_url + "payload", client_context_factory, body=s + return _load_context_factory_from_settings(crawler) + + @staticmethod + def _get_dummy_protocol() -> TLSMemoryBIOProtocol: + # from Twisted src/twisted/web/test/test_agent.py::dummyTLSProtocol() + factory = TLSMemoryBIOFactory( + optionsForClientTLS("example.com"), True, Factory.forProtocol(TxProtocol) ) + return factory.buildProtocol(None) + + @coroutine_test + async def test_payload( + self, factory: _ScrapyClientContextFactory, server_url: str + ) -> None: + s = "0123456789" * 10 + body = await self.get_page(server_url + "payload", factory, body=s) assert body == to_bytes(s) - def test_override_getContext(self): - class MyFactory(ScrapyClientContextFactory): - def getContext( - self, hostname: Any = None, port: Any = None - ) -> OpenSSL.SSL.Context: - ctx: OpenSSL.SSL.Context = super().getContext(hostname, port) - return ctx + @pytest.mark.skipif( + TWISTED_TLS_NEW_IMPL, + reason="The context is not stored on this Twisted version", + ) + def test_no_context_sharing(self, factory: _ScrapyClientContextFactory) -> None: + """Every call to creatorForNetloc() should give a fresh context.""" + creator1 = factory.creatorForNetloc(b"website1.tld", 443) + assert creator1._hostnameBytes == b"website1.tld" + creator2 = factory.creatorForNetloc(b"website2.tld", 443) + assert creator2._hostnameBytes == b"website2.tld" + assert creator1._ctx is not creator2._ctx # type: ignore[attr-defined] - with warnings.catch_warnings(record=True) as w: - MyFactory() - assert len(w) == 1 - assert ( - "Overriding ScrapyClientContextFactory.getContext() is deprecated" - in str(w[0].message) + def test_no_context_sharing_with_conn( + self, factory: _ScrapyClientContextFactory + ) -> None: + """Like test_no_context_sharing() but get the context from a connection.""" + creator1 = factory.creatorForNetloc(b"website1.tld", 443) + assert creator1._hostnameBytes == b"website1.tld" + conn1 = creator1.clientConnectionForTLS(self._get_dummy_protocol()) + + creator2 = factory.creatorForNetloc(b"website2.tld", 443) + assert creator2._hostnameBytes == b"website2.tld" + conn2 = creator2.clientConnectionForTLS(self._get_dummy_protocol()) + + assert conn1.get_context() is not conn2.get_context() + + @pytest.mark.skipif( + PYOPENSSL_SET_CIPHER_LIST_TMP_CONN, + reason="Fails or doesn't make sense on this pyOpenSSL version", + ) + def test_no_immutable_ctx_warning( + self, factory: _ScrapyClientContextFactory + ) -> None: + """There should be no pyOpenSSL context modification warning. + + pyOpenSSL < 25.1.0 doesn't produce this warning, and on 25.1.0 it's + always produced due to + https://github.com/scrapy/scrapy/issues/6859#issuecomment-4294917851. + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", + category=DeprecationWarning, + message="Attempting to mutate a Context after a Connection was created", ) + factory.creatorForNetloc(b"website.tld", 443) + + def test_ctx_flags(self, factory: _ScrapyClientContextFactory) -> None: + """The context should have the expected flags set.""" + creator = factory.creatorForNetloc(b"website.tld", 443) + conn = creator.clientConnectionForTLS(self._get_dummy_protocol()) + ctx = conn.get_context() + # fragile but pyOpenSSL doesn't have Context.get_options() + options = OpenSSL.SSL._lib.SSL_CTX_get_options(ctx._context) # type: ignore[attr-defined] + assert options & 0x4 # OP_LEGACY_SERVER_CONNECT class TestContextFactoryTLSMethod(TestContextFactoryBase): async def _assert_factory_works( - self, server_url: str, client_context_factory: ScrapyClientContextFactory + self, server_url: str, client_context_factory: _ScrapyClientContextFactory ) -> None: s = "0123456789" * 10 body = await self.get_page( @@ -138,28 +193,43 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): @coroutine_test async def test_setting_default(self, server_url: str) -> None: crawler = get_crawler() - settings = Settings() - client_context_factory = load_context_factory_from_settings(settings, crawler) + client_context_factory = _load_context_factory_from_settings(crawler) assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD await self._assert_factory_works(server_url, client_context_factory) def test_setting_none(self): - crawler = get_crawler() - settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": None}) - with pytest.raises(KeyError): - load_context_factory_from_settings(settings, crawler) + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": None}) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated", + ), + pytest.raises(KeyError), + ): + _load_context_factory_from_settings(crawler) def test_setting_bad(self): - crawler = get_crawler() - settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) - with pytest.raises(KeyError): - load_context_factory_from_settings(settings, crawler) + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated", + ), + pytest.raises(KeyError), + ): + _load_context_factory_from_settings(crawler) + @pytest.mark.filterwarnings( + r"ignore:Passing method to twisted\.internet\.ssl\.CertificateOptions:DeprecationWarning" + ) @coroutine_test async def test_setting_explicit(self, server_url: str) -> None: - crawler = get_crawler() - settings = Settings({"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) - client_context_factory = load_context_factory_from_settings(settings, crawler) + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) + with pytest.warns( + ScrapyDeprecationWarning, + match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated", + ): + client_context_factory = _load_context_factory_from_settings(crawler) assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD await self._assert_factory_works(server_url, client_context_factory) @@ -167,13 +237,18 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): async def test_direct_from_crawler(self, server_url: str) -> None: # the setting is ignored crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) - client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) + client_context_factory = build_from_crawler( + _ScrapyClientContextFactory, crawler + ) assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD await self._assert_factory_works(server_url, client_context_factory) + @pytest.mark.filterwarnings( + r"ignore:Passing method to twisted\.internet\.ssl\.CertificateOptions:DeprecationWarning" + ) @coroutine_test async def test_direct_init(self, server_url: str) -> None: - client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) + client_context_factory = _ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD await self._assert_factory_works(server_url, client_context_factory) @@ -190,3 +265,16 @@ async def test_fetch_deprecated_spider_arg(): match=r"The fetch\(\) method of .+\.CustomDownloader requires a spider argument", ): await crawler.crawl_async() + + +def test_deprecated_tls_module_names() -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match="scrapy.core.downloader.tls.METHOD_TLS is deprecated", + ): + assert tls.METHOD_TLS == "TLS" + with pytest.warns( + ScrapyDeprecationWarning, + match="scrapy.core.downloader.tls.openssl_methods is deprecated", + ): + assert isinstance(tls.openssl_methods, dict) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 7e9e029a1..be6c80429 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -8,13 +8,13 @@ from typing import TYPE_CHECKING, Any from urllib.parse import urlencode, urlparse import pytest -from testfixtures import LogCapture +from cryptography.x509 import load_der_x509_certificate from twisted.internet.defer import succeed from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from scrapy import Spider, signals -from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner +from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload from scrapy.http import Request from scrapy.http.response import Response @@ -23,7 +23,6 @@ from scrapy.utils.engine import format_engine_status, get_engine_status from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE -from tests.mockserver.http import MockServer from tests.spiders import ( AsyncDefAsyncioGenComplexSpider, AsyncDefAsyncioGenExcSpider, @@ -55,44 +54,36 @@ from tests.spiders import ( StartGoodAndBadOutput, StartItemSpider, ) -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.statscollectors import StatsCollector + from tests.mockserver.http import MockServer class TestCrawl: - mockserver: MockServer - - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - @inline_callbacks_test - def test_follow_all(self): + @coroutine_test + async def test_follow_all(self, mockserver: MockServer) -> None: crawler = get_crawler(FollowAllSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, FollowAllSpider) assert len(crawler.spider.urls_visited) == 11 # 10 + start_url @coroutine_test - async def test_fixed_delay(self): - await self._test_delay(total=3, delay=0.2) + async def test_fixed_delay(self, mockserver: MockServer) -> None: + await self._test_delay(mockserver, total=3, delay=0.2) @coroutine_test - async def test_randomized_delay(self): - await self._test_delay(total=3, delay=0.1, randomize=True) + async def test_randomized_delay(self, mockserver: MockServer) -> None: + await self._test_delay(mockserver, total=3, delay=0.1, randomize=True) + @staticmethod async def _test_delay( - self, total: int, delay: float, randomize: bool = False + mockserver: MockServer, total: int, delay: float, randomize: bool = False ) -> None: crawl_kwargs = { "maxlatency": delay * 2, - "mockserver": self.mockserver, + "mockserver": mockserver, "total": total, } tolerance = 1 - (0.6 if randomize else 0.2) @@ -121,18 +112,20 @@ class TestCrawl: average = total_time / (len(times) - 1) assert average <= delay / tolerance, "test total or delay values are too small" - @inline_callbacks_test - def test_timeout_success(self): + @coroutine_test + async def test_timeout_success(self, mockserver: MockServer) -> None: crawler = get_crawler(DelaySpider) - yield crawler.crawl(n=0.5, mockserver=self.mockserver) + await crawler.crawl_async(n=0.5, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) assert crawler.spider.t1 > 0 assert crawler.spider.t2 > 0 assert crawler.spider.t2 > crawler.spider.t1 - @inline_callbacks_test - def test_timeout_failure(self): + @coroutine_test + async def test_timeout_failure(self, mockserver: MockServer) -> None: crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) - yield crawler.crawl(n=0.5, mockserver=self.mockserver) + await crawler.crawl_async(n=0.5, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) assert crawler.spider.t1 > 0 assert crawler.spider.t2 == 0 assert crawler.spider.t2_err > 0 @@ -140,81 +133,96 @@ class TestCrawl: # server hangs after receiving response headers crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) - yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) + await crawler.crawl_async(n=0.5, b=1, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) assert crawler.spider.t1 > 0 assert crawler.spider.t2 == 0 assert crawler.spider.t2_err > 0 assert crawler.spider.t2_err > crawler.spider.t1 - @inline_callbacks_test - def test_retry_503(self): + @coroutine_test + async def test_retry_503( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=503"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/status?n=503"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_retry_conn_failed(self): + @coroutine_test + async def test_retry_conn_failed( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - "http://localhost:65432/status?n=503", mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + "http://localhost:65432/status?n=503", mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_retry_dns_error(self): + @coroutine_test + async def test_retry_dns_error( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: if NON_EXISTING_RESOLVABLE: pytest.skip("Non-existing hosts are resolvable") crawler = get_crawler(SimpleSpider) - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): # try to fetch the homepage of a nonexistent domain - yield crawler.crawl( - "http://dns.resolution.invalid./", mockserver=self.mockserver + await crawler.crawl_async( + "http://dns.resolution.invalid./", mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_start_bug_before_yield(self): - with LogCapture("scrapy", level=logging.ERROR) as log: + @coroutine_test + async def test_start_bug_before_yield( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.ERROR): crawler = get_crawler(BrokenStartSpider) - yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) + await crawler.crawl_async(fail_before_yield=1, mockserver=mockserver) - assert len(log.records) == 1 - record = log.records[0] + assert len(caplog.records) == 1 + record = caplog.records[0] assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @inline_callbacks_test - def test_start_bug_yielding(self): - with LogCapture("scrapy", level=logging.ERROR) as log: + @coroutine_test + async def test_start_bug_yielding( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.ERROR): crawler = get_crawler(BrokenStartSpider) - yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) + await crawler.crawl_async(fail_yielding=1, mockserver=mockserver) - assert len(log.records) == 1 - record = log.records[0] + assert len(caplog.records) == 1 + record = caplog.records[0] assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @inline_callbacks_test - def test_start_items(self): + @coroutine_test + async def test_start_items( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: items = [] def _on_item_scraped(item): items.append(item) - with LogCapture("scrapy", level=logging.ERROR) as log: + with caplog.at_level(logging.ERROR): crawler = get_crawler(StartItemSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) - assert len(log.records) == 0 + assert len(caplog.records) == 0 assert items == [{"name": "test item"}] - @inline_callbacks_test - def test_start_unsupported_output(self): + @coroutine_test + async def test_start_unsupported_output( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: """Anything that is not a request is assumed to be an item, avoiding a potentially expensive call to itemadapter.is_item(), and letting instead things fail when ItemAdapter is actually used on the @@ -225,35 +233,39 @@ class TestCrawl: def _on_item_scraped(item): items.append(item) - with LogCapture("scrapy", level=logging.ERROR) as log: + with caplog.at_level(logging.ERROR): crawler = get_crawler(StartGoodAndBadOutput) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) - assert len(log.records) == 0 + assert len(caplog.records) == 0 assert len(items) == 3 assert not any(isinstance(item, Request) for item in items) - @inline_callbacks_test - def test_start_dupes(self): + @coroutine_test + async def test_start_dupes(self, mockserver: MockServer) -> None: settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(DuplicateStartSpider, settings) - yield crawler.crawl( - dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver + await crawler.crawl_async( + dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=mockserver ) + assert isinstance(crawler.spider, DuplicateStartSpider) assert crawler.spider.visited == 6 crawler = get_crawler(DuplicateStartSpider, settings) - yield crawler.crawl( + await crawler.crawl_async( dont_filter=False, distinct_urls=3, dupe_factor=4, - mockserver=self.mockserver, + mockserver=mockserver, ) + assert isinstance(crawler.spider, DuplicateStartSpider) assert crawler.spider.visited == 3 - @inline_callbacks_test - def test_unbounded_response(self): + @coroutine_test + async def test_unbounded_response( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" query = urlencode( @@ -278,40 +290,45 @@ with multiples lines } ) crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url(f"/raw?{query}"), mockserver=mockserver ) - assert str(log).count("Got response 200") == 1 + assert caplog.text.count("Got response 200") == 1 - @inline_callbacks_test - def test_retry_conn_lost(self): + @coroutine_test + async def test_retry_conn_lost( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # connection lost after receiving data crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/drop?abort=0"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_retry_conn_aborted(self): + @coroutine_test + async def test_retry_conn_aborted( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # connection lost before receiving data crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/drop?abort=1"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - def _assert_retried(self, log): + @staticmethod + def _assert_retried(log: str) -> None: assert str(log).count("Retrying") == 2 assert str(log).count("Gave up retrying") == 1 - @inline_callbacks_test - def test_referer_header(self): + @coroutine_test + async def test_referer_header(self, mockserver: MockServer) -> None: """Referer header is set by RefererMiddleware unless it is already set""" - req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1) + req0 = Request(mockserver.url("/echo?headers=1&body=0"), dont_filter=True) req1 = req0.replace() req2 = req0.replace(headers={"Referer": None}) req3 = req0.replace(headers={"Referer": "http://example.com"}) @@ -319,12 +336,13 @@ with multiples lines req1.meta["next"] = req2 req2.meta["next"] = req3 crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=req0, mockserver=self.mockserver) + await crawler.crawl_async(seed=req0, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta # start() doesn't set Referer header - echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body)) + echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][0].body)) assert "Referer" not in echo0["headers"] # following request sets Referer to the source request url echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body)) @@ -336,33 +354,35 @@ with multiples lines echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) assert echo3["headers"].get("Referer") == ["http://example.com"] - @inline_callbacks_test - def test_engine_status(self): + @coroutine_test + async def test_engine_status(self, mockserver: MockServer) -> None: est = [] def cb(response): est.append(get_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl( - seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver + await crawler.crawl_async( + seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver ) + assert isinstance(crawler.spider, SingleRequestSpider) assert len(est) == 1, est s = dict(est[0]) assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == 1 - @inline_callbacks_test - def test_format_engine_status(self): + @coroutine_test + async def test_format_engine_status(self, mockserver: MockServer) -> None: est = [] def cb(response): est.append(format_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl( - seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver + await crawler.crawl_async( + seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver ) + assert isinstance(crawler.spider, SingleRequestSpider) assert len(est) == 1, est est = est[0].split("\n")[2:-2] # remove header & footer # convert to dict @@ -370,13 +390,15 @@ with multiples lines est = [x for sublist in est for x in sublist] # flatten est = [x.lstrip().rstrip() for x in est] it = iter(est) - s = dict(zip(it, it, strict=False)) + s = dict(zip(it, it, strict=True)) assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @inline_callbacks_test - def test_open_spider_error_on_faulty_pipeline(self): + @coroutine_test + async def test_open_spider_error_on_faulty_pipeline( + self, mockserver: MockServer + ) -> None: settings = { "ITEM_PIPELINES": { "tests.pipelines.ZeroDivisionErrorPipeline": 300, @@ -384,41 +406,64 @@ with multiples lines } crawler = get_crawler(SimpleSpider, settings) with pytest.raises(ZeroDivisionError): - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) assert not crawler.crawling - @inline_callbacks_test - def test_crawlerrunner_accepts_crawler(self): - crawler = get_crawler(SimpleSpider) - runner = CrawlerRunner() - with LogCapture() as log: - yield runner.crawl( - crawler, - self.mockserver.url("/status?n=200"), - mockserver=self.mockserver, + @coroutine_test + async def test_open_spider_error_on_faulty_pipeline_crawl( + self, mockserver: MockServer + ) -> None: + # cover the except block in Crawler.crawl() + settings = { + "ITEM_PIPELINES": { + "tests.pipelines.ZeroDivisionErrorPipeline": 300, + } + } + crawler = get_crawler(SimpleSpider, settings) + with pytest.raises(ZeroDivisionError): + await maybe_deferred_to_future( + crawler.crawl(mockserver.url("/status?n=200"), mockserver=mockserver) ) - assert "Got response 200" in str(log) + assert not crawler.crawling @coroutine_test - async def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture) -> None: + async def test_crawlerrunner_accepts_crawler( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + crawler = Crawler(SimpleSpider, get_reactor_settings()) + runner = CrawlerRunner() + with caplog.at_level(logging.DEBUG): + await maybe_deferred_to_future( + runner.crawl( + crawler, + mockserver.url("/status?n=200"), + mockserver=mockserver, + ) + ) + assert "Got response 200" in caplog.text + + @coroutine_test + async def test_crawl_multiple( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: settings_dict = get_reactor_settings() runner_cls = ( CrawlerRunner - if settings_dict.get("TWISTED_ENABLED", True) + if settings_dict.get("TWISTED_REACTOR_ENABLED", True) else AsyncCrawlerRunner ) runner = runner_cls(settings_dict) runner.crawl( SimpleSpider, - self.mockserver.url("/status?n=200"), - mockserver=self.mockserver, + mockserver.url("/status?n=200"), + mockserver=mockserver, ) runner.crawl( SimpleSpider, - self.mockserver.url("/status?n=503"), - mockserver=self.mockserver, + mockserver.url("/status?n=503"), + mockserver=mockserver, ) with caplog.at_level(logging.DEBUG): @@ -430,25 +475,15 @@ with multiples lines @coroutine_test async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler(SimpleSpider) - await maybe_deferred_to_future(crawler.crawl("foo://bar")) + await crawler.crawl_async("foo://bar") assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text class TestCrawlSpider: - mockserver: MockServer - - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - + @staticmethod async def _run_spider( - self, spider_cls: type[Spider] - ) -> tuple[LogCapture, list[Any], StatsCollector]: + spider_cls: type[Spider], mockserver: MockServer + ) -> tuple[list[Any], StatsCollector]: items = [] def _on_item_scraped(item): @@ -456,103 +491,119 @@ class TestCrawlSpider: crawler = get_crawler(spider_cls) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - await maybe_deferred_to_future( - crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver - ) - ) + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver + ) assert crawler.stats - return log, items, crawler.stats + return items, crawler.stats - @inline_callbacks_test - def test_crawlspider_with_parse(self): + @coroutine_test + async def test_crawlspider_with_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithParseMethod) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse] status 200 (foo: None)" in str(log) - assert "[parse] status 201 (foo: None)" in str(log) - assert "[parse] status 202 (foo: bar)" in str(log) + assert "[parse] status 200 (foo: None)" in caplog.text + assert "[parse] status 201 (foo: None)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_crawlspider_with_async_callback(self): + @coroutine_test + async def test_crawlspider_with_async_callback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithAsyncCallback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse_async] status 200 (foo: None)" in str(log) - assert "[parse_async] status 201 (foo: None)" in str(log) - assert "[parse_async] status 202 (foo: bar)" in str(log) + assert "[parse_async] status 200 (foo: None)" in caplog.text + assert "[parse_async] status 201 (foo: None)" in caplog.text + assert "[parse_async] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_crawlspider_with_async_generator_callback(self): + @coroutine_test + async def test_crawlspider_with_async_generator_callback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse_async_gen] status 200 (foo: None)" in str(log) - assert "[parse_async_gen] status 201 (foo: None)" in str(log) - assert "[parse_async_gen] status 202 (foo: bar)" in str(log) + assert "[parse_async_gen] status 200 (foo: None)" in caplog.text + assert "[parse_async_gen] status 201 (foo: None)" in caplog.text + assert "[parse_async_gen] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_crawlspider_with_errback(self): + @coroutine_test + async def test_crawlspider_with_errback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithErrback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse] status 200 (foo: None)" in str(log) - assert "[parse] status 201 (foo: None)" in str(log) - assert "[parse] status 202 (foo: bar)" in str(log) - assert "[errback] status 404" in str(log) - assert "[errback] status 500" in str(log) - assert "[errback] status 501" in str(log) + assert "[parse] status 200 (foo: None)" in caplog.text + assert "[parse] status 201 (foo: None)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text + assert "[errback] status 404" in caplog.text + assert "[errback] status 500" in caplog.text + assert "[errback] status 501" in caplog.text - @inline_callbacks_test - def test_crawlspider_process_request_cb_kwargs(self): + @coroutine_test + async def test_crawlspider_process_request_cb_kwargs( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse] status 200 (foo: process_request)" in str(log) - assert "[parse] status 201 (foo: process_request)" in str(log) - assert "[parse] status 202 (foo: bar)" in str(log) + assert "[parse] status 200 (foo: process_request)" in caplog.text + assert "[parse] status 201 (foo: process_request)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_async_def_parse(self): + @coroutine_test + async def test_async_def_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(AsyncDefSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - assert "Got response 200" in str(log) + assert "Got response 200" in caplog.text @pytest.mark.only_asyncio - @inline_callbacks_test - def test_async_def_asyncio_parse(self): + @coroutine_test + async def test_async_def_asyncio_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler( AsyncDefAsyncioSpider, { "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor" }, ) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - assert "Got response 200" in str(log) + assert "Got response 200" in caplog.text @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncio_parse_items_list(self): - log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider) - assert "Got response 200" in str(log) + async def test_async_def_asyncio_parse_items_list( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider, mockserver) + assert "Got response 200" in caplog.text assert {"id": 1} in items assert {"id": 2} in items @pytest.mark.only_asyncio - @inline_callbacks_test - def test_async_def_asyncio_parse_items_single_element(self): + @coroutine_test + async def test_async_def_asyncio_parse_items_single_element( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: items = [] def _on_item_scraped(item): @@ -560,26 +611,34 @@ class TestCrawlSpider: crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - assert "Got response 200" in str(log) + assert "Got response 200" in caplog.text assert {"foo": 42} in items @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse(self): - log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider) - assert "Got response 200" in str(log) + async def test_async_def_asyncgen_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + _, stats = await self._run_spider(AsyncDefAsyncioGenSpider, mockserver) + assert "Got response 200" in caplog.text itemcount = stats.get_value("item_scraped_count") assert itemcount == 1 @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse_loop(self): - log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider) - assert "Got response 200" in str(log) + async def test_async_def_asyncgen_parse_loop( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, stats = await self._run_spider( + AsyncDefAsyncioGenLoopSpider, mockserver + ) + assert "Got response 200" in caplog.text itemcount = stats.get_value("item_scraped_count") assert itemcount == 10 for i in range(10): @@ -587,11 +646,15 @@ class TestCrawlSpider: @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse_exc(self): - log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider) - log = str(log) - assert "Spider error processing" in log - assert "ValueError" in log + async def test_async_def_asyncgen_parse_exc( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, stats = await self._run_spider( + AsyncDefAsyncioGenExcSpider, mockserver + ) + assert "Spider error processing" in caplog.text + assert "ValueError" in caplog.text itemcount = stats.get_value("item_scraped_count") assert itemcount == 7 for i in range(7): @@ -599,8 +662,12 @@ class TestCrawlSpider: @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse_complex(self): - _, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider) + async def test_async_def_asyncgen_parse_complex( + self, mockserver: MockServer + ) -> None: + items, stats = await self._run_spider( + AsyncDefAsyncioGenComplexSpider, mockserver + ) itemcount = stats.get_value("item_scraped_count") assert itemcount == 156 # some random items @@ -611,39 +678,45 @@ class TestCrawlSpider: @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncio_parse_reqs_list(self): - log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider) + async def test_async_def_asyncio_parse_reqs_list( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + await self._run_spider(AsyncDefAsyncioReqsReturnSpider, mockserver) for req_id in range(3): - assert f"Got response 200, req_id {req_id}" in str(log) + assert f"Got response 200, req_id {req_id}" in caplog.text @pytest.mark.only_not_asyncio @coroutine_test - async def test_async_def_deferred_direct(self): - _, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider) + async def test_async_def_deferred_direct(self, mockserver: MockServer) -> None: + items, _ = await self._run_spider(AsyncDefDeferredDirectSpider, mockserver) assert items == [{"code": 200}] @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_deferred_wrapped(self): - _, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider) + async def test_async_def_deferred_wrapped(self, mockserver: MockServer) -> None: + items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider, mockserver) assert items == [{"code": 200}] @coroutine_test - async def test_async_def_deferred_maybe_wrapped(self): - _, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider) + async def test_async_def_deferred_maybe_wrapped( + self, mockserver: MockServer + ) -> None: + items, _ = await self._run_spider( + AsyncDefDeferredMaybeWrappedSpider, mockserver + ) assert items == [{"code": 200}] - @inline_callbacks_test - def test_response_ssl_certificate_none(self): + @coroutine_test + async def test_response_ssl_certificate_none(self, mockserver: MockServer) -> None: crawler = get_crawler(SingleRequestSpider) - url = self.mockserver.url("/echo?body=test", is_secure=False) - yield crawler.crawl(seed=url, mockserver=self.mockserver) + url = mockserver.url("/echo?body=test", is_secure=False) + await crawler.crawl_async(seed=url, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) assert crawler.spider.meta["responses"][0].certificate is None - @pytest.mark.xfail( - 'config.getoption("--reactor") == "none"', - reason="Not implemented in HttpxDownloadHandler", - strict=True, + @pytest.mark.filterwarnings( + r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning" ) @pytest.mark.parametrize( "url", @@ -668,9 +741,14 @@ class TestCrawlSpider: await crawler.crawl_async(seed=url, mockserver=mockserver) assert isinstance(crawler.spider, SingleRequestSpider) cert = crawler.spider.meta["responses"][0].certificate - assert isinstance(cert, Certificate) - assert cert.getSubject().commonName == b"localhost" - assert cert.getIssuer().commonName == b"localhost" + assert cert is not None + if isinstance(cert, Certificate): # Twisted + assert cert.getSubject().commonName == b"localhost" + assert cert.getIssuer().commonName == b"localhost" + elif isinstance(cert, bytes): # DER bytes + cert_x509 = load_der_x509_certificate(cert) + assert cert_x509.subject.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" + assert cert_x509.issuer.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" @pytest.mark.parametrize( "url", @@ -697,10 +775,13 @@ class TestCrawlSpider: assert isinstance(ip_address, IPv4Address) assert str(ip_address) == gethostbyname(expected_netloc) - @inline_callbacks_test - def test_bytes_received_stop_download_callback(self): + @coroutine_test + async def test_bytes_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(BytesReceivedCallbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, BytesReceivedCallbackSpider) assert crawler.spider.meta.get("failure") is None assert isinstance(crawler.spider.meta["response"], Response) assert crawler.spider.meta["response"].body == crawler.spider.meta.get( @@ -711,10 +792,13 @@ class TestCrawlSpider: < crawler.spider.full_response_length ) - @inline_callbacks_test - def test_bytes_received_stop_download_errback(self): + @coroutine_test + async def test_bytes_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(BytesReceivedErrbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, BytesReceivedErrbackSpider) assert crawler.spider.meta.get("response") is None assert isinstance(crawler.spider.meta["failure"], Failure) assert isinstance(crawler.spider.meta["failure"].value, StopDownload) @@ -727,20 +811,26 @@ class TestCrawlSpider: < crawler.spider.full_response_length ) - @inline_callbacks_test - def test_headers_received_stop_download_callback(self): + @coroutine_test + async def test_headers_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(HeadersReceivedCallbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, HeadersReceivedCallbackSpider) assert crawler.spider.meta.get("failure") is None assert isinstance(crawler.spider.meta["response"], Response) assert crawler.spider.meta["response"].headers == crawler.spider.meta.get( "headers_received" ) - @inline_callbacks_test - def test_headers_received_stop_download_errback(self): + @coroutine_test + async def test_headers_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(HeadersReceivedErrbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, HeadersReceivedErrbackSpider) assert crawler.spider.meta.get("response") is None assert isinstance(crawler.spider.meta["failure"], Failure) assert isinstance(crawler.spider.meta["failure"].value, StopDownload) @@ -749,8 +839,10 @@ class TestCrawlSpider: "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") - @inline_callbacks_test - def test_spider_callback_deferred_deprecated(self): + @coroutine_test + async def test_spider_callback_deferred_deprecated( + self, mockserver: MockServer + ) -> None: def cb(response: Response) -> Any: return succeed(None) @@ -759,10 +851,12 @@ class TestCrawlSpider: ScrapyDeprecationWarning, match="Returning Deferreds from spider callbacks is deprecated", ): - yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) - @inline_callbacks_test - def test_spider_errback(self): + @coroutine_test + async def test_spider_errback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: failures = [] def eb(failure: Failure) -> Failure: @@ -770,72 +864,82 @@ class TestCrawlSpider: return failure crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) assert len(failures) == 1 - assert "HTTP status code is not handled or not allowed" in str(log) - assert "Spider error processing" not in str(log) + assert "HTTP status code is not handled or not allowed" in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_spider_errback_silence(self): + @coroutine_test + async def test_spider_errback_silence( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: failures = [] def eb(failure: Failure) -> None: failures.append(failure) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) assert len(failures) == 1 - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_spider_errback_exception(self): + @coroutine_test + async def test_spider_errback_exception( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> None: raise ValueError("foo") crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - assert "Spider error processing" in str(log) + assert "Spider error processing" in caplog.text - @inline_callbacks_test - def test_spider_errback_item(self): + @coroutine_test + async def test_spider_errback_item( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Any: return {"foo": "bar"} crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "'item_scraped_count': 1" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "'item_scraped_count': 1" in caplog.text - @inline_callbacks_test - def test_spider_errback_request(self): + @coroutine_test + async def test_spider_errback_request( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Request: - return Request(self.mockserver.url("/")) + return Request(mockserver.url("/")) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "Crawled (200)" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "Crawled (200)" in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error(self): + @coroutine_test + async def test_spider_errback_downloader_error( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: failures = [] def eb(failure: Failure) -> Failure: @@ -843,57 +947,65 @@ class TestCrawlSpider: return failure crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) assert len(failures) == 1 - assert "Error downloading" in str(log) - assert "Spider error processing" not in str(log) + assert "Error downloading" in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error_exception(self): + @coroutine_test + async def test_spider_errback_downloader_error_exception( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> None: raise ValueError("foo") crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) - assert "Error downloading" in str(log) - assert "Spider error processing" in str(log) + assert "Error downloading" in caplog.text + assert "Spider error processing" in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error_item(self): + @coroutine_test + async def test_spider_errback_downloader_error_item( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Any: return {"foo": "bar"} crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "'item_scraped_count': 1" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "'item_scraped_count': 1" in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error_request(self): + @coroutine_test + async def test_spider_errback_downloader_error_request( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Request: - return Request(self.mockserver.url("/")) + return Request(mockserver.url("/")) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "Crawled (200)" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "Crawled (200)" in caplog.text - @inline_callbacks_test - def test_spider_errback_deferred_deprecated(self): + @coroutine_test + async def test_spider_errback_deferred_deprecated( + self, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Any: return succeed(None) @@ -902,28 +1014,32 @@ class TestCrawlSpider: ScrapyDeprecationWarning, match="Returning Deferreds from spider errbacks is deprecated", ): - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - @inline_callbacks_test - def test_raise_closespider(self): + @coroutine_test + async def test_raise_closespider( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def cb(response): raise CloseSpider crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) - assert "Closing spider (cancelled)" in str(log) - assert "Spider error processing" not in str(log) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) + assert "Closing spider (cancelled)" in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_raise_closespider_reason(self): + @coroutine_test + async def test_raise_closespider_reason( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def cb(response): raise CloseSpider("my_reason") crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) - assert "Closing spider (my_reason)" in str(log) - assert "Spider error processing" not in str(log) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) + assert "Closing spider (my_reason)" in caplog.text + assert "Spider error processing" not in caplog.text diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 9ade8472f..0cddfd0ed 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,21 +1,12 @@ +from __future__ import annotations + import asyncio import logging -import platform import re -import signal -import subprocess -import sys -import warnings -from abc import ABC, abstractmethod -from collections.abc import Generator from pathlib import Path -from typing import Any +from typing import Any, ClassVar import pytest -from packaging.version import parse as parse_version -from pexpect.popen_spawn import PopenSpawn -from twisted.internet.defer import Deferred -from w3lib import __version__ as w3lib_version from zope.interface.exceptions import MultipleInvalid import scrapy @@ -26,12 +17,12 @@ from scrapy.crawler import ( Crawler, CrawlerProcess, CrawlerRunner, + CrawlerRunnerBase, ) from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings -from scrapy.utils.asyncio import call_later -from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future from scrapy.utils.log import ( _uninstall_scrapy_root_handler, configure_logging, @@ -39,14 +30,14 @@ from scrapy.utils.log import ( ) from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings -from tests.mockserver.http import MockServer -from tests.utils import get_script_run_env -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.decorators import coroutine_test BASE_SETTINGS: dict[str, Any] = {} -def get_raw_crawler(spidercls=None, settings_dict=None): +def get_raw_crawler( + spidercls: type[Spider] | None = None, settings_dict: dict[str, Any] | None = None +) -> Crawler: """get_crawler alternative that only calls the __init__ method of the crawler.""" settings = Settings() @@ -56,14 +47,18 @@ def get_raw_crawler(spidercls=None, settings_dict=None): class TestBaseCrawler: - def assertOptionIsDefault(self, settings, key): + @staticmethod + def assertOptionIsDefault(settings: Settings, key: str) -> None: assert isinstance(settings, Settings) assert settings[key] == getattr(default_settings, key) class TestCrawler(TestBaseCrawler): - def test_populate_spidercls_settings(self): - spider_settings = {"TEST1": "spider", "TEST2": "spider"} + def test_populate_spidercls_settings(self) -> None: + spider_settings: dict[str, Any] = { + "TEST1": "spider", + "TEST2": "spider", + } project_settings = { **BASE_SETTINGS, "TEST1": "project", @@ -86,47 +81,45 @@ class TestCrawler(TestBaseCrawler): assert not settings.frozen assert crawler.settings.frozen - def test_crawler_accepts_dict(self): + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar" self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") - def test_crawler_accepts_None(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - crawler = Crawler(DefaultSpider) + def test_crawler_accepts_None(self) -> None: + crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") - def test_crawler_rejects_spider_objects(self): + def test_crawler_rejects_spider_objects(self) -> None: with pytest.raises(ValueError, match="spidercls argument must be a class"): - Crawler(DefaultSpider()) - - @inline_callbacks_test - def test_crawler_crawl_twice_seq_unsupported(self): - crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) - yield crawler.crawl() - with pytest.raises(RuntimeError, match="more than once on the same instance"): - yield crawler.crawl() + Crawler(DefaultSpider()) # type: ignore[arg-type] @coroutine_test - async def test_crawler_crawl_async_twice_seq_unsupported(self): + async def test_crawler_crawl_twice_seq_unsupported(self) -> None: + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + await maybe_deferred_to_future(crawler.crawl()) + with pytest.raises(RuntimeError, match="more than once on the same instance"): + await maybe_deferred_to_future(crawler.crawl()) + + @coroutine_test + async def test_crawler_crawl_async_twice_seq_unsupported(self) -> None: crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) await crawler.crawl_async() with pytest.raises(RuntimeError, match="more than once on the same instance"): await crawler.crawl_async() - @inline_callbacks_test - def test_crawler_crawl_twice_parallel_unsupported(self): + @coroutine_test + async def test_crawler_crawl_twice_parallel_unsupported(self) -> None: crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) d1 = crawler.crawl() d2 = crawler.crawl() - yield d1 + await maybe_deferred_to_future(d1) with pytest.raises(RuntimeError, match="Crawling already taking place"): - yield d2 + await maybe_deferred_to_future(d2) @pytest.mark.only_asyncio @coroutine_test - async def test_crawler_crawl_async_twice_parallel_unsupported(self): + async def test_crawler_crawl_async_twice_parallel_unsupported(self) -> None: crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) t1 = asyncio.create_task(crawler.crawl_async()) t2 = asyncio.create_task(crawler.crawl_async()) @@ -134,12 +127,12 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError, match="Crawling already taking place"): await t2 - def test_get_addon(self): + def test_get_addon(self) -> None: class ParentAddon: pass class TrackingAddon(ParentAddon): - instances = [] + instances: ClassVar[list[TrackingAddon]] = [] def __init__(self): TrackingAddon.instances.append(self) @@ -160,7 +153,7 @@ class TestCrawler(TestBaseCrawler): addon = crawler.get_addon(TrackingAddon) assert addon == expected - addon = crawler.get_addon(DefaultSpider) + addon = crawler.get_addon(DefaultSpider) # type: ignore[assignment] assert addon is None addon = crawler.get_addon(ParentAddon) @@ -172,19 +165,21 @@ class TestCrawler(TestBaseCrawler): addon = crawler.get_addon(ChildAddon) assert addon is None - @inline_callbacks_test - def test_get_downloader_middleware(self): + @coroutine_test + async def test_get_downloader_middleware(self) -> None: class ParentDownloaderMiddleware: pass class TrackingDownloaderMiddleware(ParentDownloaderMiddleware): - instances = [] + instances: ClassVar[list[TrackingDownloaderMiddleware]] = [] def __init__(self): TrackingDownloaderMiddleware.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -208,18 +203,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingDownloaderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingDownloaderMiddleware.instances) == 1 assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentDownloaderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] class ChildDownloaderMiddleware(TrackingDownloaderMiddleware): @@ -227,44 +222,42 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildDownloaderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_downloader_middleware_not_crawling(self): + def test_get_downloader_middleware_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_downloader_middleware(DefaultSpider) - @inline_callbacks_test - def test_get_downloader_middleware_no_engine(self): + @coroutine_test + async def test_get_downloader_middleware_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @classmethod def from_crawler(cls, crawler): - try: - crawler.get_downloader_middleware(DefaultSpider) - except Exception as e: - MySpider.result = e - raise + crawler.get_downloader_middleware(DefaultSpider) crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() - @inline_callbacks_test - def test_get_extension(self): + @coroutine_test + async def test_get_extension(self) -> None: class ParentExtension: pass class TrackingExtension(ParentExtension): - instances = [] + instances: ClassVar[list[TrackingExtension]] = [] def __init__(self): TrackingExtension.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -288,18 +281,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingExtension - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingExtension.instances) == 1 assert MySpider.result == TrackingExtension.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentExtension - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingExtension.instances[-1] class ChildExtension(TrackingExtension): @@ -307,44 +300,42 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildExtension - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_extension_not_crawling(self): + def test_get_extension_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_extension(DefaultSpider) - @inline_callbacks_test - def test_get_extension_no_engine(self): + @coroutine_test + async def test_get_extension_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @classmethod def from_crawler(cls, crawler): - try: - crawler.get_extension(DefaultSpider) - except Exception as e: - MySpider.result = e - raise + crawler.get_extension(DefaultSpider) crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() - @inline_callbacks_test - def test_get_item_pipeline(self): + @coroutine_test + async def test_get_item_pipeline(self) -> None: class ParentItemPipeline: pass class TrackingItemPipeline(ParentItemPipeline): - instances = [] + instances: ClassVar[list[TrackingItemPipeline]] = [] def __init__(self): TrackingItemPipeline.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -368,18 +359,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingItemPipeline - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingItemPipeline.instances) == 1 assert MySpider.result == TrackingItemPipeline.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentItemPipeline - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingItemPipeline.instances[-1] class ChildItemPipeline(TrackingItemPipeline): @@ -387,44 +378,42 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildItemPipeline - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_item_pipeline_not_crawling(self): + def test_get_item_pipeline_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_item_pipeline(DefaultSpider) - @inline_callbacks_test - def test_get_item_pipeline_no_engine(self): + @coroutine_test + async def test_get_item_pipeline_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @classmethod def from_crawler(cls, crawler): - try: - crawler.get_item_pipeline(DefaultSpider) - except Exception as e: - MySpider.result = e - raise + crawler.get_item_pipeline(DefaultSpider) crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() - @inline_callbacks_test - def test_get_spider_middleware(self): + @coroutine_test + async def test_get_spider_middleware(self) -> None: class ParentSpiderMiddleware: pass class TrackingSpiderMiddleware(ParentSpiderMiddleware): - instances = [] + instances: ClassVar[list[TrackingSpiderMiddleware]] = [] def __init__(self): TrackingSpiderMiddleware.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -448,18 +437,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingSpiderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingSpiderMiddleware.instances) == 1 assert MySpider.result == TrackingSpiderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentSpiderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingSpiderMiddleware.instances[-1] class ChildSpiderMiddleware(TrackingSpiderMiddleware): @@ -467,45 +456,42 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildSpiderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_spider_middleware_not_crawling(self): + def test_get_spider_middleware_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_spider_middleware(DefaultSpider) - @inline_callbacks_test - def test_get_spider_middleware_no_engine(self): + @coroutine_test + async def test_get_spider_middleware_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @classmethod def from_crawler(cls, crawler): - try: - crawler.get_spider_middleware(DefaultSpider) - except Exception as e: - MySpider.result = e - raise + crawler.get_spider_middleware(DefaultSpider) crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() class TestSpiderSettings: - def test_spider_custom_settings(self): + def test_spider_custom_settings(self) -> None: class MySpider(scrapy.Spider): name = "spider" custom_settings = {"AUTOTHROTTLE_ENABLED": True} crawler = get_crawler(MySpider) + assert crawler.extensions enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] assert AutoThrottle in enabled_exts class TestCrawlerLogging: - def test_no_root_handler_installed(self): + def test_no_root_handler_installed(self) -> None: handler = get_scrapy_root_handler() if handler is not None: logging.root.removeHandler(handler) @@ -517,7 +503,7 @@ class TestCrawlerLogging: assert get_scrapy_root_handler() is None @coroutine_test - async def test_spider_custom_settings_log_level(self, tmp_path): + async def test_spider_custom_settings_log_level(self, tmp_path: Path) -> None: log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") @@ -545,9 +531,13 @@ class TestCrawlerLogging: try: configure_logging() - assert get_scrapy_root_handler().level == logging.DEBUG + handler = get_scrapy_root_handler() + assert handler is not None + assert handler.level == logging.DEBUG crawler = get_crawler(MySpider) - assert get_scrapy_root_handler().level == logging.INFO + handler = get_scrapy_root_handler() + assert handler is not None + assert handler.level == logging.INFO await crawler.crawl_async() finally: _uninstall_scrapy_root_handler() @@ -559,12 +549,13 @@ class TestCrawlerLogging: assert "info message" in logged assert "warning message" in logged assert "error message" in logged + assert crawler.stats assert crawler.stats.get_value("log_count/ERROR") == 1 assert crawler.stats.get_value("log_count/WARNING") == 1 assert info_count == 1 assert crawler.stats.get_value("log_count/DEBUG", 0) == 0 - def test_spider_custom_settings_log_append(self, tmp_path): + def test_spider_custom_settings_log_append(self, tmp_path: Path) -> None: log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") @@ -589,12 +580,12 @@ class TestCrawlerLogging: class SpiderLoaderWithWrongInterface: - def unneeded_method(self): + def unneeded_method(self) -> None: pass class TestCrawlerRunner(TestBaseCrawler): - def test_spider_manager_verify_interface(self): + def test_spider_manager_verify_interface(self) -> None: settings = Settings( { "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, @@ -603,18 +594,18 @@ class TestCrawlerRunner(TestBaseCrawler): with pytest.raises(MultipleInvalid): CrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self): + def test_crawler_runner_accepts_dict(self) -> None: runner = CrawlerRunner({"foo": "bar"}) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_runner_accepts_None(self): + def test_crawler_runner_accepts_None(self) -> None: runner = CrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") class TestAsyncCrawlerRunner(TestBaseCrawler): - def test_spider_manager_verify_interface(self): + def test_spider_manager_verify_interface(self) -> None: settings = Settings( { "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, @@ -623,23 +614,23 @@ class TestAsyncCrawlerRunner(TestBaseCrawler): with pytest.raises(MultipleInvalid): AsyncCrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self): + def test_crawler_runner_accepts_dict(self) -> None: runner = AsyncCrawlerRunner({"foo": "bar"}) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_runner_accepts_None(self): + def test_crawler_runner_accepts_None(self) -> None: runner = AsyncCrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") class TestCrawlerProcess(TestBaseCrawler): - def test_crawler_process_accepts_dict(self): + def test_crawler_process_accepts_dict(self) -> None: runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_process_accepts_None(self): + def test_crawler_process_accepts_None(self) -> None: runner = CrawlerProcess(install_root_handler=False) self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") @@ -648,18 +639,54 @@ class TestCrawlerProcess(TestBaseCrawler): class TestAsyncCrawlerProcess(TestBaseCrawler): def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None: runner = AsyncCrawlerProcess( - {"foo": "bar", "TWISTED_ENABLED": reactor_pytest != "none"}, + {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"}, install_root_handler=False, ) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - @pytest.mark.requires_reactor # can't pass TWISTED_ENABLED=False + @pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False def test_crawler_process_accepts_None(self) -> None: runner = AsyncCrawlerProcess(install_root_handler=False) self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_runner_settings_applied_to_crawler_instance( + runner_cls: type[CrawlerRunnerBase], +) -> None: + runner = runner_cls({"FOO": "runner"}) + crawler = Crawler(DefaultSpider) + result = runner.create_crawler(crawler) + assert result is crawler + assert result.settings["FOO"] == "runner" + + +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_spider_custom_settings_override_runner( + runner_cls: type[CrawlerRunnerBase], +) -> None: + class MySpider(DefaultSpider): + custom_settings = {"FOO": "spider"} + + runner = runner_cls({"FOO": "runner"}) + crawler = Crawler(MySpider) + runner.create_crawler(crawler) + assert crawler.settings["FOO"] == "spider" + + +def test_create_crawler_instance_consistent_with_spider_class() -> None: + runner = AsyncCrawlerRunner({"FOO": "runner"}) + + crawler_from_class = runner.create_crawler(DefaultSpider) + + pre_built = Crawler(DefaultSpider) + runner.create_crawler(pre_built) + + assert crawler_from_class.settings["FOO"] == "runner" + assert pre_built.settings["FOO"] == "runner" + + class ExceptionSpider(scrapy.Spider): name = "exception" @@ -676,35 +703,37 @@ class NoRequestsSpider(scrapy.Spider): yield -@pytest.mark.requires_reactor +@pytest.mark.requires_reactor # CrawlerRunner requires a reactor class TestCrawlerRunnerHasSpider: - @staticmethod - def _runner(): + @pytest.fixture + def runner(self) -> CrawlerRunnerBase: return CrawlerRunner(get_reactor_settings()) @staticmethod - def _crawl(runner, spider): - return runner.crawl(spider) + async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None: + await ensure_awaitable(runner.crawl(spider)) - @inline_callbacks_test - def test_crawler_runner_bootstrap_successful(self): - runner = self._runner() - yield self._crawl(runner, NoRequestsSpider) + @coroutine_test + async def test_crawler_runner_bootstrap_successful( + self, runner: CrawlerRunnerBase + ) -> None: + await self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_bootstrap_successful_for_several(self): - runner = self._runner() - yield self._crawl(runner, NoRequestsSpider) - yield self._crawl(runner, NoRequestsSpider) + @coroutine_test + async def test_crawler_runner_bootstrap_successful_for_several( + self, runner: CrawlerRunnerBase + ) -> None: + await self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_bootstrap_failed(self): - runner = self._runner() - + @coroutine_test + async def test_crawler_runner_bootstrap_failed( + self, runner: CrawlerRunnerBase + ) -> None: try: - yield self._crawl(runner, ExceptionSpider) + await self._crawl(runner, ExceptionSpider) except ValueError: pass else: @@ -712,25 +741,25 @@ class TestCrawlerRunnerHasSpider: assert runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_bootstrap_failed_for_several(self): - runner = self._runner() - + @coroutine_test + async def test_crawler_runner_bootstrap_failed_for_several( + self, runner: CrawlerRunnerBase + ) -> None: try: - yield self._crawl(runner, ExceptionSpider) + await self._crawl(runner, ExceptionSpider) except ValueError: pass else: pytest.fail("Exception should be raised from spider") - yield self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) assert runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_asyncio_enabled_true( + @coroutine_test + async def test_crawler_runner_asyncio_enabled_true( self, reactor_pytest: str - ) -> Generator[Deferred[Any], Any, None]: + ) -> None: if reactor_pytest != "asyncio": runner = CrawlerRunner( settings={ @@ -741,537 +770,26 @@ class TestCrawlerRunnerHasSpider: Exception, match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", ): - yield self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) else: - CrawlerRunner( + runner = CrawlerRunner( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", } ) + await self._crawl(runner, NoRequestsSpider) @pytest.mark.only_asyncio class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): - @staticmethod - def _runner(): + @pytest.fixture + def runner(self) -> CrawlerRunnerBase: return AsyncCrawlerRunner(get_reactor_settings()) - @staticmethod - def _crawl(runner, spider): - return deferred_from_coro(runner.crawl(spider)) - - def test_crawler_runner_asyncio_enabled_true(self): + def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override] pytest.skip("This test is only for CrawlerRunner") -class ScriptRunnerMixin(ABC): - @property - @abstractmethod - def script_dir(self) -> Path: - raise NotImplementedError - - @staticmethod - def get_script_dir(name: str) -> Path: - return Path(__file__).parent.resolve() / name - - def get_script_args(self, script_name: str, *script_args: str) -> list[str]: - script_path = self.script_dir / script_name - return [sys.executable, str(script_path), *script_args] - - def run_script(self, script_name: str, *script_args: str) -> str: - args = self.get_script_args(script_name, *script_args) - p = subprocess.Popen( - args, - env=get_script_run_env(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - _, stderr = p.communicate() - return stderr.decode("utf-8") - - -class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): - """Common tests between CrawlerProcess and AsyncCrawlerProcess, - with the same file names and expectations. - """ - - def test_simple(self): - log = self.run_script("simple.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "is_reactorless(): False" in log - - def test_multi(self): - log = self.run_script("multi.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "ReactorAlreadyInstalledError" not in log - - def test_reactor_default(self): - log = self.run_script("reactor_default.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" - ) in log - - def test_asyncio_enabled_no_reactor(self): - log = self.run_script("asyncio_enabled_no_reactor.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "RuntimeError" not in log - - def test_asyncio_enabled_reactor(self): - log = self.run_script("asyncio_enabled_reactor.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "RuntimeError" not in log - - @pytest.mark.skipif( - parse_version(w3lib_version) >= parse_version("2.0.0"), - reason="w3lib 2.0.0 and later do not allow invalid domains.", - ) - def test_ipv6_default_name_resolver(self): - log = self.run_script("default_name_resolver.py") - assert "Spider closed (finished)" in log - assert ( - "'downloader/exception_type_count/scrapy.exceptions.CannotResolveHostError': 1," - in log - ) - assert ( - "scrapy.exceptions.CannotResolveHostError: DNS lookup failed: no results for hostname lookup: ::1." - in log - ) - - def test_caching_hostname_resolver_ipv6(self): - log = self.run_script("caching_hostname_resolver_ipv6.py") - assert "Spider closed (finished)" in log - assert "scrapy.exceptions.CannotResolveHostError" not in log - - def test_caching_hostname_resolver_finite_execution( - self, mockserver: MockServer - ) -> None: - log = self.run_script("caching_hostname_resolver.py", mockserver.url("/")) - assert "Spider closed (finished)" in log - assert "ERROR: Error downloading" not in log - assert "TimeoutError" not in log - assert "scrapy.exceptions.CannotResolveHostError" not in log - - def test_twisted_reactor_asyncio(self): - log = self.run_script("twisted_reactor_asyncio.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - def test_twisted_reactor_asyncio_custom_settings(self): - log = self.run_script("twisted_reactor_custom_settings.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - def test_twisted_reactor_asyncio_custom_settings_same(self): - log = self.run_script("twisted_reactor_custom_settings_same.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - @pytest.mark.requires_uvloop - def test_custom_loop_asyncio(self): - log = self.run_script("asyncio_custom_loop.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - @pytest.mark.requires_uvloop - def test_custom_loop_asyncio_deferred_signal(self): - log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" in log - assert "async pipeline opened!" in log - - @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_same_loop(self): - log = self.run_script("asyncio_enabled_reactor_same_loop.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_different_loop(self): - log = self.run_script("asyncio_enabled_reactor_different_loop.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the one specified in the ASYNCIO_EVENT_LOOP " - "setting (uvloop.Loop)" - ) in log - - def test_default_loop_asyncio_deferred_signal(self): - log = self.run_script("asyncio_deferred_signal.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" not in log - assert "async pipeline opened!" in log - - def test_args_change_settings(self): - log = self.run_script("args_settings.py") - assert "Spider closed (finished)" in log - assert "The value of FOO is 42" in log - - def test_shutdown_graceful(self): - sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "3") - p = PopenSpawn(args, timeout=5, env=get_script_run_env()) - p.expect_exact("Spider opened") - p.expect_exact("Crawled (200)") - p.kill(sig) - p.expect_exact("shutting down gracefully") - p.expect_exact("Spider closed (shutdown)") - p.wait() - - @inline_callbacks_test - def test_shutdown_forced(self): - sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "10") - p = PopenSpawn(args, timeout=5, env=get_script_run_env()) - p.expect_exact("Spider opened") - p.expect_exact("Crawled (200)") - p.kill(sig) - p.expect_exact("shutting down gracefully") - # sending the second signal too fast often causes problems - d = Deferred() - call_later(0.01, d.callback, None) - yield d - p.kill(sig) - p.expect_exact("forcing unclean shutdown") - p.wait() - - -class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): - @property - def script_dir(self) -> Path: - return self.get_script_dir("CrawlerProcess") - - def test_reactor_default_twisted_reactor_select(self): - log = self.run_script("reactor_default_twisted_reactor_select.py") - if platform.system() in ["Windows", "Darwin"]: - # The goal of this test function is to test that, when a reactor is - # installed (the default one here) and a different reactor is - # configured (select here), an error raises. - # - # In Windows the default reactor is the select reactor, so that - # error does not raise. - # - # If that ever becomes the case on more platforms (i.e. if Linux - # also starts using the select reactor by default in a future - # version of Twisted), then we will need to rethink this test. - assert "Spider closed (finished)" in log - else: - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - - def test_reactor_select(self): - log = self.run_script("reactor_select.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" - ) in log - - def test_reactor_select_twisted_reactor_select(self): - log = self.run_script("reactor_select_twisted_reactor_select.py") - assert "Spider closed (finished)" in log - assert "ReactorAlreadyInstalledError" not in log - - def test_reactor_select_subclass_twisted_reactor_select(self): - log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - - def test_twisted_reactor_select(self): - log = self.run_script("twisted_reactor_select.py") - assert "Spider closed (finished)" in log - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - - @pytest.mark.skipif( - platform.system() == "Windows", reason="PollReactor is not supported on Windows" - ) - def test_twisted_reactor_poll(self): - log = self.run_script("twisted_reactor_poll.py") - assert "Spider closed (finished)" in log - assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log - - def test_twisted_reactor_asyncio_custom_settings_conflict(self): - log = self.run_script("twisted_reactor_custom_settings_conflict.py") - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - assert ( - "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" - in log - ) - - def test_reactorless(self): - log = self.run_script("reactorless.py") - assert ( - "RuntimeError: CrawlerProcess doesn't support TWISTED_ENABLED=False" in log - ) - - -class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): - @property - def script_dir(self) -> Path: - return self.get_script_dir("AsyncCrawlerProcess") - - def test_twisted_reactor_custom_settings_select(self): - log = self.run_script("twisted_reactor_custom_settings_select.py") - assert "Spider closed (finished)" not in log - assert ( - "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) " - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - - @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_same_loop(self): - log = self.run_script("asyncio_custom_loop_custom_settings_same.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_different_loop(self): - log = self.run_script("asyncio_custom_loop_custom_settings_different.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the one specified in the ASYNCIO_EVENT_LOOP " - "setting (uvloop.Loop)" - ) in log - - def test_reactorless_simple(self): - log = self.run_script("reactorless_simple.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "is_reactorless(): True" in log - assert "ERROR: " not in log - assert "WARNING: " not in log - - def test_reactorless_datauri(self): - log = self.run_script("reactorless_datauri.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "{'data': 'foo'}" in log - assert "'item_scraped_count': 1" in log - assert "ERROR: " not in log - assert "WARNING: " not in log - - def test_reactorless_import_hook(self): - log = self.run_script("reactorless_import_hook.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "ImportError: Import of twisted.internet.reactor is forbidden" in log - - def test_reactorless_telnetconsole_default(self): - """By default TWISTED_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False.""" - log = self.run_script("reactorless_telnetconsole_default.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "The TelnetConsole extension requires a Twisted reactor" not in log - assert "scrapy.extensions.telnet.TelnetConsole" not in log - - def test_reactorless_telnetconsole_disabled(self): - """Explicit TELNETCONSOLE_ENABLED=False, there are no warnings.""" - log = self.run_script("reactorless_telnetconsole_disabled.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "The TelnetConsole extension requires a Twisted reactor" not in log - assert "scrapy.extensions.telnet.TelnetConsole" not in log - - def test_reactorless_telnetconsole_enabled(self): - """Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning.""" - log = self.run_script("reactorless_telnetconsole_enabled.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "The TelnetConsole extension requires a Twisted reactor" in log - - def test_reactorless_reactor(self): - log = self.run_script("reactorless_reactor.py") - assert ( - "RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed" - in log - ) - - -class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): - """Common tests between CrawlerRunner and AsyncCrawlerRunner, - with the same file names and expectations. - """ - - def test_simple(self): - log = self.run_script("simple.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "is_reactorless(): False" in log - - def test_multi_parallel(self): - log = self.run_script("multi_parallel.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert re.search( - r"Spider opened.+Spider opened.+Closing spider.+Closing spider", - log, - re.DOTALL, - ) - - def test_multi_seq(self): - log = self.run_script("multi_seq.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert re.search( - r"Spider opened.+Closing spider.+Spider opened.+Closing spider", - log, - re.DOTALL, - ) - - @pytest.mark.requires_uvloop - def test_custom_loop_same(self): - log = self.run_script("custom_loop_same.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - @pytest.mark.requires_uvloop - def test_custom_loop_different(self): - log = self.run_script("custom_loop_different.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the one specified in the ASYNCIO_EVENT_LOOP " - "setting (uvloop.Loop)" - ) in log - - -class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): - @property - def script_dir(self) -> Path: - return self.get_script_dir("CrawlerRunner") - - def test_explicit_default_reactor(self): - log = self.run_script("explicit_default_reactor.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - - def test_response_ip_address(self): - log = self.run_script("ip_address.py") - assert "INFO: Spider closed (finished)" in log - assert "INFO: Host: not.a.real.domain" in log - assert "INFO: Type: " in log - assert "INFO: IP address: 127.0.0.1" in log - - def test_change_default_reactor(self): - log = self.run_script("change_reactor.py") - assert ( - "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert "DEBUG: Using asyncio event loop" in log - - def test_reactorless(self): - log = self.run_script("reactorless.py") - assert ( - "RuntimeError: CrawlerRunner doesn't support TWISTED_ENABLED=False" in log - ) - - -class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): - @property - def script_dir(self) -> Path: - return self.get_script_dir("AsyncCrawlerRunner") - - def test_simple_default_reactor(self): - log = self.run_script("simple_default_reactor.py") - assert "Spider closed (finished)" not in log - assert ( - "RuntimeError: When TWISTED_ENABLED is True, " - "AsyncCrawlerRunner requires that the installed Twisted reactor" - ) in log - - def test_reactorless_simple(self): - log = self.run_script("reactorless_simple.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "is_reactorless(): True" in log - assert "ERROR: " not in log - assert "WARNING: " not in log - - def test_reactorless_datauri(self): - log = self.run_script("reactorless_datauri.py") - assert "Not using a Twisted reactor" in log - assert "Spider closed (finished)" in log - assert "{'data': 'foo'}" in log - assert "'item_scraped_count': 1" in log - assert "ERROR: " not in log - assert "WARNING: " not in log - - def test_reactorless_reactor(self): - log = self.run_script("reactorless_reactor.py") - assert ( - "RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed" - in log - ) - - @pytest.mark.parametrize( ("settings", "items"), [ @@ -1280,7 +798,9 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): ({"LOG_VERSIONS": []}, None), ], ) -def test_log_scrapy_info(settings, items, caplog): +def test_log_scrapy_info( + settings: dict[str, Any], items: list[str] | None, caplog: pytest.LogCaptureFixture +) -> None: with caplog.at_level("INFO"): CrawlerProcess(settings, install_root_handler=False) assert ( diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py new file mode 100644 index 000000000..146d94ecd --- /dev/null +++ b/tests/test_crawler_subprocess.py @@ -0,0 +1,575 @@ +from __future__ import annotations + +import platform +import re +import signal +import subprocess +import sys +from abc import ABC, abstractmethod +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +from packaging.version import parse as parse_version +from pexpect.popen_spawn import PopenSpawn +from w3lib import __version__ as w3lib_version + +from tests.utils import async_sleep, get_script_run_env +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer + + +class ScriptRunnerMixin(ABC): + @property + @abstractmethod + def script_dir(self) -> Path: + raise NotImplementedError + + @staticmethod + def get_script_dir(name: str) -> Path: + return Path(__file__).parent.resolve() / name + + def get_script_args(self, script_name: str, *script_args: str) -> list[str]: + script_path = self.script_dir / script_name + return [sys.executable, str(script_path), *script_args] + + def run_script(self, script_name: str, *script_args: str) -> str: + args = self.get_script_args(script_name, *script_args) + p = subprocess.Popen( + args, + env=get_script_run_env(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + _, stderr = p.communicate() + return stderr.decode("utf-8") + + +class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): + """Common tests between CrawlerProcess and AsyncCrawlerProcess, + with the same file names and expectations. + """ + + def test_simple(self) -> None: + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "is_reactorless(): False" in log + + def test_multi(self) -> None: + log = self.run_script("multi.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "ReactorAlreadyInstalledError" not in log + + def test_reactor_default(self) -> None: + log = self.run_script("reactor_default.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log + + def test_asyncio_enabled_no_reactor(self) -> None: + log = self.run_script("asyncio_enabled_no_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "RuntimeError" not in log + + def test_asyncio_enabled_reactor(self) -> None: + log = self.run_script("asyncio_enabled_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "RuntimeError" not in log + + @pytest.mark.skipif( + parse_version(w3lib_version) >= parse_version("2.0.0"), + reason="w3lib 2.0.0 and later do not allow invalid domains.", + ) + def test_ipv6_default_name_resolver(self) -> None: + log = self.run_script("default_name_resolver.py") + assert "Spider closed (finished)" in log + assert ( + "'downloader/exception_type_count/scrapy.exceptions.CannotResolveHostError': 1," + in log + ) + assert ( + "scrapy.exceptions.CannotResolveHostError: DNS lookup failed: no results for hostname lookup: ::1." + in log + ) + + def test_caching_hostname_resolver_ipv6(self) -> None: + log = self.run_script("caching_hostname_resolver_ipv6.py") + assert "Spider closed (finished)" in log + assert "scrapy.exceptions.CannotResolveHostError" not in log + + def test_caching_hostname_resolver_finite_execution( + self, mockserver: MockServer + ) -> None: + log = self.run_script("caching_hostname_resolver.py", mockserver.url("/")) + assert "Spider closed (finished)" in log + assert "ERROR: Error downloading" not in log + assert "TimeoutError" not in log + assert "scrapy.exceptions.CannotResolveHostError" not in log + + def test_twisted_reactor_asyncio(self) -> None: + log = self.run_script("twisted_reactor_asyncio.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_twisted_reactor_asyncio_custom_settings(self) -> None: + log = self.run_script("twisted_reactor_custom_settings.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_twisted_reactor_asyncio_custom_settings_same(self) -> None: + log = self.run_script("twisted_reactor_custom_settings_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + @pytest.mark.requires_uvloop + def test_custom_loop_asyncio(self) -> None: + log = self.run_script("asyncio_custom_loop.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_custom_loop_asyncio_deferred_signal(self) -> None: + log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + assert "async pipeline opened!" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_same_loop(self) -> None: + log = self.run_script("asyncio_enabled_reactor_same_loop.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_different_loop(self) -> None: + log = self.run_script("asyncio_enabled_reactor_different_loop.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + def test_default_loop_asyncio_deferred_signal(self) -> None: + log = self.run_script("asyncio_deferred_signal.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" not in log + assert "async pipeline opened!" in log + + def test_args_change_settings(self) -> None: + log = self.run_script("args_settings.py") + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log + + def _test_shutdown_graceful(self, script: str = "sleeping.py") -> None: + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined] + args = self.get_script_args(script, "3") + p = PopenSpawn(args, timeout=5, env=get_script_run_env()) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + p.expect_exact("Spider closed (shutdown)") + p.wait() # type: ignore[no-untyped-call] + + def test_shutdown_graceful(self) -> None: + self._test_shutdown_graceful() + + async def _test_shutdown_forced(self, script: str = "sleeping.py") -> None: + sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined] + args = self.get_script_args(script, "10") + p = PopenSpawn(args, timeout=5, env=get_script_run_env()) + p.expect_exact("Spider opened") + p.expect_exact("Crawled (200)") + p.kill(sig) + p.expect_exact("shutting down gracefully") + # sending the second signal too fast often causes problems + await async_sleep(0.01) + p.kill(sig) + p.expect_exact("forcing unclean shutdown") + p.wait() # type: ignore[no-untyped-call] + + @coroutine_test + async def test_shutdown_forced(self) -> None: + await self._test_shutdown_forced() + + +class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerProcess") + + def test_reactor_default_twisted_reactor_select(self) -> None: + log = self.run_script("reactor_default_twisted_reactor_select.py") + if platform.system() in ["Windows", "Darwin"]: + # The goal of this test function is to test that, when a reactor is + # installed (the default one here) and a different reactor is + # configured (select here), an error raises. + # + # In Windows the default reactor is the select reactor, so that + # error does not raise. + # + # If that ever becomes the case on more platforms (i.e. if Linux + # also starts using the select reactor by default in a future + # version of Twisted), then we will need to rethink this test. + assert "Spider closed (finished)" in log + else: + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_reactor_select(self) -> None: + log = self.run_script("reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log + + def test_reactor_select_twisted_reactor_select(self) -> None: + log = self.run_script("reactor_select_twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "ReactorAlreadyInstalledError" not in log + + def test_reactor_select_subclass_twisted_reactor_select(self) -> None: + log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_twisted_reactor_select(self) -> None: + log = self.run_script("twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + + @pytest.mark.skipif( + platform.system() == "Windows", reason="PollReactor is not supported on Windows" + ) + def test_twisted_reactor_poll(self) -> None: + log = self.run_script("twisted_reactor_poll.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log + + def test_twisted_reactor_asyncio_custom_settings_conflict(self) -> None: + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" + in log + ) + + def test_reactorless(self) -> None: + log = self.run_script("reactorless.py") + assert ( + "RuntimeError: CrawlerProcess doesn't support TWISTED_REACTOR_ENABLED=False" + in log + ) + + +class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerProcess") + + def test_twisted_reactor_custom_settings_select(self) -> None: + log = self.run_script("twisted_reactor_custom_settings_select.py") + assert "Spider closed (finished)" not in log + assert ( + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) " + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_same_loop(self) -> None: + log = self.run_script("asyncio_custom_loop_custom_settings_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_different_loop(self) -> None: + log = self.run_script("asyncio_custom_loop_custom_settings_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + def test_reactorless_simple(self) -> None: + log = self.run_script("reactorless_simple.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "is_reactorless(): True" in log + assert "ERROR: " not in log + assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 + assert log.count("WARNING: ") == 2 + + def test_reactorless_custom_settings(self) -> None: + """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not + currently supported, AsyncCrawlerProcess will install a reactor in this + case. + """ + log = self.run_script("reactorless_custom_settings.py") + assert "Spider closed (finished)" not in log + assert ( + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." + in log + ) + + def test_reactorless_datauri(self) -> None: + log = self.run_script("reactorless_datauri.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "{'data': 'foo'}" in log + assert "'item_scraped_count': 1" in log + assert "ERROR: " not in log + assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 + assert log.count("WARNING: ") == 2 + + def test_reactorless_import_hook(self) -> None: + log = self.run_script("reactorless_import_hook.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "ImportError: Import of twisted.internet.reactor is forbidden" in log + + def test_reactorless_telnetconsole_default(self) -> None: + """By default TWISTED_REACTOR_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False.""" + log = self.run_script("reactorless_simple.py") # no need for a separate script + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" not in log + assert "scrapy.extensions.telnet.TelnetConsole" not in log + + def test_reactorless_telnetconsole_disabled(self) -> None: + """Explicit TELNETCONSOLE_ENABLED=False, there are no warnings.""" + log = self.run_script("reactorless_telnetconsole_disabled.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" not in log + assert "scrapy.extensions.telnet.TelnetConsole" not in log + + def test_reactorless_telnetconsole_enabled(self) -> None: + """Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning.""" + log = self.run_script("reactorless_telnetconsole_enabled.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" in log + + def test_reactorless_reactor(self) -> None: + log = self.run_script("reactorless_reactor.py") + assert ( + "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed" + in log + ) + + def test_shutdown_graceful(self) -> None: + self._test_shutdown_graceful("reactorless_sleeping.py") + + @coroutine_test + async def test_shutdown_forced(self) -> None: + await self._test_shutdown_forced("reactorless_sleeping.py") + + +class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): + """Common tests between CrawlerRunner and AsyncCrawlerRunner, + with the same file names and expectations. + """ + + def test_simple(self) -> None: + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "is_reactorless(): False" in log + + def test_multi_parallel(self) -> None: + log = self.run_script("multi_parallel.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Spider opened.+Closing spider.+Closing spider", + log, + re.DOTALL, + ) + + def test_multi_seq(self) -> None: + log = self.run_script("multi_seq.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Closing spider.+Spider opened.+Closing spider", + log, + re.DOTALL, + ) + + @pytest.mark.requires_uvloop + def test_custom_loop_same(self) -> None: + log = self.run_script("custom_loop_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_custom_loop_different(self) -> None: + log = self.run_script("custom_loop_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + def test_no_reactor(self) -> None: + log = self.run_script("no_reactor.py") + assert "Spider closed (finished)" not in log + assert ( + "RuntimeError: We expected a Twisted reactor to be installed but it isn't." + in log + ) + + +class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerRunner") + + def test_explicit_default_reactor(self) -> None: + log = self.run_script("explicit_default_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + + def test_response_ip_address(self) -> None: + log = self.run_script("ip_address.py") + assert "INFO: Spider closed (finished)" in log + assert "INFO: Host: not.a.real.domain" in log + assert "INFO: Type: " in log + assert "INFO: IP address: 127.0.0.1" in log + + def test_change_default_reactor(self) -> None: + log = self.run_script("change_reactor.py") + assert ( + "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "DEBUG: Using asyncio event loop" in log + + def test_reactorless(self) -> None: + log = self.run_script("reactorless.py") + assert ( + "RuntimeError: CrawlerRunner doesn't support TWISTED_REACTOR_ENABLED=False" + in log + ) + + +class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerRunner") + + def test_simple_default_reactor(self) -> None: + log = self.run_script("simple_default_reactor.py") + assert "Spider closed (finished)" not in log + assert ( + "RuntimeError: When TWISTED_REACTOR_ENABLED is True, " + "AsyncCrawlerRunner requires that the installed Twisted reactor" + ) in log + + def test_reactorless_simple(self) -> None: + log = self.run_script("reactorless_simple.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "is_reactorless(): True" in log + assert "ERROR: " not in log + assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 + assert log.count("WARNING: ") == 2 + + def test_reactorless_custom_settings(self) -> None: + """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not + currently supported, AsyncCrawlerRunner will expect a reactor installed + by the user. + """ + log = self.run_script("reactorless_custom_settings.py") + assert "Spider closed (finished)" not in log + assert "We expected a Twisted reactor to be installed but it isn't." in log + + def test_reactorless_datauri(self) -> None: + log = self.run_script("reactorless_datauri.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "{'data': 'foo'}" in log + assert "'item_scraped_count': 1" in log + assert "ERROR: " not in log + assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 + assert log.count("WARNING: ") == 2 + + def test_reactorless_reactor(self) -> None: + log = self.run_script("reactorless_reactor.py") + assert ( + "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed" + in log + ) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 4436efd9b..578e84359 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -15,14 +15,14 @@ class TestScrapyUtils: See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 """ - if not os.environ.get("_SCRAPY_PINNED", None): - pytest.skip("Not in a pinned environment") + if not os.environ.get("_SCRAPY_MIN", None): + pytest.skip("Not in a min environment") tox_config_file_path = Path(__file__).parent / ".." / "tox.ini" config_parser = ConfigParser() config_parser.read(tox_config_file_path) pattern = r"Twisted==([\d.]+)" - match = re.search(pattern, config_parser["pinned"]["deps"]) + match = re.search(pattern, config_parser["min"]["deps"]) pinned_twisted_version_string = match[1] assert twisted_version.short() == pinned_twisted_version_string diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index ad6c6d3c7..fdacad963 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -2,20 +2,30 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +import sys +from typing import TYPE_CHECKING, Any, ClassVar import pytest from scrapy import Request +from scrapy.core.downloader.handlers._httpx import ( + HAS_HTTP2, + HAS_SOCKS, + HttpxDownloadHandler, +) +from scrapy.exceptions import DownloadFailedError from tests.test_downloader_handlers_http_base import ( - TestHttp11Base, + TestHttpBase, TestHttpProxyBase, - TestHttps11Base, + TestHttpsBase, TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, + TestHttpsTLSVersionBase, TestHttpsWrongHostnameBase, TestHttpWithCrawlerBase, + TestMitmProxyBase, + TestRealWebsiteBase, TestSimpleHttpsBase, ) from tests.utils.decorators import coroutine_test @@ -25,79 +35,16 @@ if TYPE_CHECKING: from tests.mockserver.http import MockServer +pytestmark = pytest.mark.only_asyncio + pytest.importorskip("httpx") class HttpxDownloadHandlerMixin: @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - # the import will fail if httpx is not installed - from scrapy.core.downloader.handlers._httpx import ( # noqa: PLC0415 - HttpxDownloadHandler, - ) - return HttpxDownloadHandler - -class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base): - @coroutine_test - async def test_unsupported_bindaddress( - self, caplog: pytest.LogCaptureFixture, mockserver: MockServer - ) -> None: - meta = {"bindaddress": "127.0.0.2"} - request = Request(mockserver.url("/text"), meta=meta) - async with self.get_dh() as download_handler: - response = await download_handler.download_request(request) - assert response.body == b"Works" - assert ( - "The 'bindaddress' request meta key is not supported by HttpxDownloadHandler" - in caplog.text - ) - - @coroutine_test - async def test_unsupported_proxy( - self, caplog: pytest.LogCaptureFixture, mockserver: MockServer - ) -> None: - meta = {"proxy": "127.0.0.2"} - request = Request(mockserver.url("/text"), meta=meta) - async with self.get_dh() as download_handler: - response = await download_handler.download_request(request) - assert response.body == b"Works" - assert ( - "The 'proxy' request meta key is not supported by HttpxDownloadHandler" - in caplog.text - ) - - -class TestHttps11(HttpxDownloadHandlerMixin, TestHttps11Base): - tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher" - - -class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase): - pass - - -class Https11WrongHostnameTestCase( - HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase -): - pass - - -class Https11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase): - pass - - -class Https11InvalidDNSPattern( - HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase -): - pass - - -class Https11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase): - pass - - -class TestHttp11WithCrawler(TestHttpWithCrawlerBase): @property def settings_dict(self) -> dict[str, Any] | None: return { @@ -108,20 +55,107 @@ class TestHttp11WithCrawler(TestHttpWithCrawlerBase): } -class TestHttps11WithCrawler(TestHttp11WithCrawler): - is_secure = True +class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase): + handler_supports_bindaddress_meta = False - @pytest.mark.skip(reason="response.certificate is not implemented") + @pytest.mark.skipif( + sys.platform == "darwin", + reason="127.0.0.2 is not available on macOS by default", + ) @coroutine_test - async def test_response_ssl_certificate(self, mockserver: MockServer) -> None: + async def test_bind_address_port_warning( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + request = Request(mockserver.url("/client-ip")) + async with self.get_dh( + {"DOWNLOAD_BIND_ADDRESS": ("127.0.0.2", 12345)} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"127.0.0.2" + assert "DOWNLOAD_BIND_ADDRESS specifies a port (12345)" in caplog.text + assert "Ignoring the port" in caplog.text + + +class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase): + handler_supports_bindaddress_meta = False + tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher" + + @pytest.mark.skip(reason="The check is Twisted-specific") + def test_verify_certs_deprecated(self) -> None: # type: ignore[override] pass -@pytest.mark.skip(reason="Proxy support is not implemented yet") -class TestHttp11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): +@pytest.mark.skipif(not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler") +class TestHttp2(TestHttps): + http2 = True + handler_supports_http2_dataloss = False + + default_handler_settings: ClassVar[dict[str, Any]] = { + "HTTPX_HTTP2_ENABLED": True, + } + + @coroutine_test + async def test_protocol(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/host", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.protocol == "HTTP/2" + + @coroutine_test + async def test_data_loss_handling(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + with pytest.raises(DownloadFailedError): + await download_handler.download_request(request) + + +class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase): pass -@pytest.mark.skip(reason="Proxy support is not implemented yet") -class TestHttps11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): +class TestHttpsWrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase): + pass + + +class TestHttpsInvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass + + +class TestHttpsInvalidDNSPattern( + HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): + pass + + +class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass + + +class TestHttpsTLSVersion(HttpxDownloadHandlerMixin, TestHttpsTLSVersionBase): + pass + + +class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase): + pass + + +class TestHttpsWithCrawler(TestHttpWithCrawler): is_secure = True + + +class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): + expected_http_proxy_request_body = b"http://example.com/" + + +class TestHttpsProxy(TestHttpProxy): + is_secure = True + + +@pytest.mark.requires_mitmproxy +class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase): + handler_supports_socks = HAS_SOCKS + + +@pytest.mark.requires_internet +class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase): + pass diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 957ad35c9..489b70e74 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -11,7 +11,10 @@ import pytest from pytest_twisted import async_yield_fixture from twisted.cred import checkers, credentials, portal +from scrapy import Spider from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler +from scrapy.crawler import Crawler +from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Request, Response from scrapy.http.response.text import TextResponse from scrapy.utils.defer import deferred_f_from_coro_f @@ -25,7 +28,7 @@ if TYPE_CHECKING: from twisted.protocols.ftp import FTPFactory -pytestmark = pytest.mark.requires_reactor +pytestmark = pytest.mark.requires_reactor # FTPDownloadHandler requires a reactor class TestFTPBase(ABC): @@ -56,7 +59,7 @@ class TestFTPBase(ABC): port = reactor.listenTCP(0, factory, interface="127.0.0.1") portno = port.getHost().port - yield f"https://127.0.0.1:{portno}/" + yield f"ftp://127.0.0.1:{portno}/" await port.stopListening() @@ -139,15 +142,11 @@ class TestFTPBase(ABC): server_url: str, dh: FTPDownloadHandler, ) -> None: - f, local_fname = mkstemp() - local_fname_path = Path(local_fname) - os.close(f) meta = {} meta.update(self.req_meta) request = Request(url=server_url + filename, meta=meta) r = await dh.download_request(request) assert type(r) is response_class # pylint: disable=unidiomatic-typecheck - local_fname_path.unlink() class TestFTP(TestFTPBase): @@ -200,3 +199,9 @@ class TestAnonymousFTP(TestFTPBase): p = portal.Portal(realm) p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) return FTPFactory(portal=p, userAnonymous=self.username) + + +def test_not_configured_without_reactor() -> None: + crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) + with pytest.raises(NotConfigured): + FTPDownloadHandler.from_crawler(crawler) diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py deleted file mode 100644 index 99f3697f8..000000000 --- a/tests/test_downloader_handler_twisted_http10.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Tests for scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pytest - -from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler -from scrapy.http import Request -from scrapy.utils.defer import deferred_f_from_coro_f -from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase - -if TYPE_CHECKING: - from scrapy.core.downloader.handlers import DownloadHandlerProtocol - from tests.mockserver.http import MockServer - - -pytestmark = pytest.mark.requires_reactor - - -class HTTP10DownloadHandlerMixin: - @property - def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - return HTTP10DownloadHandler - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase): - """HTTP 1.0 test case""" - - def test_unsupported_scheme(self) -> None: # type: ignore[override] - pytest.skip("Check not implemented") - - @deferred_f_from_coro_f - async def test_protocol(self, mockserver: MockServer) -> None: - request = Request( - mockserver.url("/host", is_secure=self.is_secure), method="GET" - ) - async with self.get_dh() as download_handler: - response = await download_handler.download_request(request) - assert response.protocol == "HTTP/1.0" - - -class TestHttps10(TestHttp10): - is_secure = True - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestHttp10Proxy(HTTP10DownloadHandlerMixin, TestHttpProxyBase): - @deferred_f_from_coro_f - async def test_download_with_proxy_https_timeout(self): - pytest.skip("Not implemented") - - @deferred_f_from_coro_f - async def test_download_with_proxy_without_http_scheme(self): - pytest.skip("Not implemented") diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index ffaa2c569..79750a136 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -2,20 +2,27 @@ from __future__ import annotations +import sys from typing import TYPE_CHECKING, Any import pytest +from scrapy import Spider from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler +from scrapy.crawler import Crawler +from scrapy.exceptions import NotConfigured from tests.test_downloader_handlers_http_base import ( - TestHttp11Base, + TestHttpBase, TestHttpProxyBase, - TestHttps11Base, + TestHttpsBase, TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, + TestHttpsTLSVersionBase, TestHttpsWrongHostnameBase, TestHttpWithCrawlerBase, + TestMitmProxyBase, + TestRealWebsiteBase, TestSimpleHttpsBase, ) @@ -23,7 +30,7 @@ if TYPE_CHECKING: from scrapy.core.downloader.handlers import DownloadHandlerProtocol -pytestmark = pytest.mark.requires_reactor +pytestmark = pytest.mark.requires_reactor # HTTP11DownloadHandler requires a reactor class HTTP11DownloadHandlerMixin: @@ -31,38 +38,6 @@ class HTTP11DownloadHandlerMixin: def download_handler_cls(self) -> type[DownloadHandlerProtocol]: return HTTP11DownloadHandler - -class TestHttp11(HTTP11DownloadHandlerMixin, TestHttp11Base): - pass - - -class TestHttps11(HTTP11DownloadHandlerMixin, TestHttps11Base): - pass - - -class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): - pass - - -class TestHttps11WrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase): - pass - - -class TestHttps11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): - pass - - -class TestHttps11InvalidDNSPattern( - HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase -): - pass - - -class TestHttps11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): - pass - - -class TestHttp11WithCrawler(TestHttpWithCrawlerBase): @property def settings_dict(self) -> dict[str, Any] | None: return { @@ -73,9 +48,72 @@ class TestHttp11WithCrawler(TestHttpWithCrawlerBase): } -class TestHttps11WithCrawler(TestHttp11WithCrawler): +def test_not_configured_without_reactor() -> None: + crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) + with pytest.raises(NotConfigured): + HTTP11DownloadHandler.from_crawler(crawler) + + +class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase): + pass + + +class TestHttps(HTTP11DownloadHandlerMixin, TestHttpsBase): + pass + + +class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase): + pass + + +class TestHttpsWrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase): + pass + + +class TestHttpsInvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass + + +class TestHttpsInvalidDNSPattern( + HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase +): + pass + + +class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase): + pass + + +class TestHttpsTLSVersion(HTTP11DownloadHandlerMixin, TestHttpsTLSVersionBase): + pass + + +class TestHttpWithCrawler(HTTP11DownloadHandlerMixin, TestHttpWithCrawlerBase): + pass + + +class TestHttpsWithCrawler(TestHttpWithCrawler): is_secure = True -class TestHttp11Proxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): +class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): pass + + +class TestHttpsProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): + is_secure = True + # not implemented + handler_supports_tls_in_tls = False + + +@pytest.mark.requires_mitmproxy +class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase): + # not implemented + handler_supports_tls_in_tls = False + + +@pytest.mark.requires_internet +class TestRealWebsite(HTTP11DownloadHandlerMixin, TestRealWebsiteBase): + @property + def platform_cert_store_works(self) -> bool: + return sys.platform != "win32" diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 1b9dc21e7..bea97642e 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -2,34 +2,39 @@ from __future__ import annotations -import json +import sys from typing import TYPE_CHECKING, Any import pytest from testfixtures import LogCapture from twisted.web.http import H2_ENABLED -from scrapy.exceptions import UnsupportedURLSchemeError +from scrapy import Spider +from scrapy.crawler import Crawler +from scrapy.exceptions import DownloadFailedError, NotConfigured from scrapy.http import Request -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from tests.test_downloader_handlers_http_base import ( TestHttpProxyBase, - TestHttps11Base, + TestHttpsBase, TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, + TestHttpsTLSVersionBase, TestHttpsWrongHostnameBase, TestHttpWithCrawlerBase, + TestMitmProxyBase, + TestRealWebsiteBase, + TestSimpleHttpsBase, ) +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.core.downloader.handlers import DownloadHandlerProtocol from tests.mockserver.http import MockServer - from tests.mockserver.proxy_echo import ProxyEchoMockServer pytestmark = [ - pytest.mark.requires_reactor, + pytest.mark.requires_reactor, # H2DownloadHandler requires a reactor pytest.mark.skipif( not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" ), @@ -46,31 +51,35 @@ class H2DownloadHandlerMixin: return H2DownloadHandler + @property + def settings_dict(self) -> dict[str, Any] | None: + return { + "DOWNLOAD_HANDLERS": { + "http": None, + "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", + } + } -class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): - HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" - @deferred_f_from_coro_f +def test_not_configured_without_reactor() -> None: + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler # noqa: PLC0415 + + crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) + with pytest.raises(NotConfigured): + H2DownloadHandler.from_crawler(crawler) + + +class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): + http2 = True + handler_supports_http2_dataloss = False + + @coroutine_test async def test_protocol(self, mockserver: MockServer) -> None: - request = Request( - mockserver.url("/host", is_secure=self.is_secure), method="GET" - ) + request = Request(mockserver.url("/host", is_secure=self.is_secure)) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.protocol == "h2" - def test_download_cause_data_loss(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_cause_data_loss_double_warning(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_allow_data_loss(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_allow_data_loss_via_setting(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - def test_download_conn_failed(self) -> None: # type: ignore[override] # Unlike HTTP11DownloadHandler which raises it from download_request() # (without any special handling), here ConnectionRefusedError (raised in @@ -79,12 +88,6 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): # DOWNLOAD_TIMEOUT. pytest.skip("The handler doesn't properly reraise ConnectionRefusedError") - def test_download_conn_lost(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - - def test_download_conn_aborted(self) -> None: # type: ignore[override] - pytest.skip(self.HTTP2_DATALOSS_SKIP_REASON) - def test_download_dns_error(self) -> None: # type: ignore[override] # Unlike HTTP11DownloadHandler which raises it from download_request() # (without any special handling), here DNSLookupError (raised in @@ -93,7 +96,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): # DOWNLOAD_TIMEOUT. pytest.skip("The handler doesn't properly reraise DNSLookupError") - @deferred_f_from_coro_f + @coroutine_test async def test_concurrent_requests_same_domain( self, mockserver: MockServer ) -> None: @@ -108,7 +111,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): assert response2.headers["Content-Length"] == b"79" @pytest.mark.xfail(reason="https://github.com/python-hyper/h2/issues/1247") - @deferred_f_from_coro_f + @coroutine_test async def test_connect_request(self, mockserver: MockServer) -> None: request = Request( mockserver.url("/file", is_secure=self.is_secure), method="CONNECT" @@ -117,7 +120,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): response = await download_handler.download_request(request) assert response.body == b"" - @deferred_f_from_coro_f + @coroutine_test async def test_custom_content_length_good(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/contentlength", is_secure=self.is_secure)) custom_content_length = str(len(request.body)) @@ -126,7 +129,7 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): response = await download_handler.download_request(request) assert response.text == custom_content_length - @deferred_f_from_coro_f + @coroutine_test async def test_custom_content_length_bad(self, mockserver: MockServer) -> None: request = Request(mockserver.url("/contentlength", is_secure=self.is_secure)) actual_content_length = str(len(request.body)) @@ -146,47 +149,41 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): ) ) - @deferred_f_from_coro_f - async def test_duplicate_header(self, mockserver: MockServer) -> None: - request = Request(mockserver.url("/echo", is_secure=self.is_secure)) - header, value1, value2 = "Custom-Header", "foo", "bar" - request.headers.appendlist(header, value1) - request.headers.appendlist(header, value2) + @coroutine_test + async def test_data_loss_handling(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) async with self.get_dh() as download_handler: - response = await download_handler.download_request(request) - assert json.loads(response.text)["headers"][header] == [value1, value2] + with pytest.raises(DownloadFailedError): + await download_handler.download_request(request) -class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): +class TestSimpleHttp2(H2DownloadHandlerMixin, TestSimpleHttpsBase): pass -class TestHttps2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): +class TestHttp2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase): pass -class TestHttps2InvalidDNSPattern( +class TestHttp2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase): + pass + + +class TestHttp2InvalidDNSPattern( H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase ): pass -class TestHttps2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): +class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase): pass -class TestHttp2WithCrawler(TestHttpWithCrawlerBase): - """HTTP 2.0 test case with MockServer""" +class TestHttp2TLSVersion(H2DownloadHandlerMixin, TestHttpsTLSVersionBase): + pass - @property - def settings_dict(self) -> dict[str, Any] | None: - return { - "DOWNLOAD_HANDLERS": { - "http": None, - "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", - } - } +class TestHttp2WithCrawler(H2DownloadHandlerMixin, TestHttpWithCrawlerBase): is_secure = True def test_bytes_received_stop_download_callback(self) -> None: # type: ignore[override] @@ -202,24 +199,19 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase): pytest.skip("headers_received support is not implemented") -class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): +@pytest.mark.skip(reason="Proxy support is not implemented yet") +class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): is_secure = True - expected_http_proxy_request_body = b"/" - @deferred_f_from_coro_f - async def test_download_with_proxy_https_timeout( - self, proxy_mockserver: ProxyEchoMockServer - ) -> None: - with pytest.raises(NotImplementedError): - await maybe_deferred_to_future( - super().test_download_with_proxy_https_timeout(proxy_mockserver) # type: ignore[arg-type] - ) - @deferred_f_from_coro_f - async def test_download_with_proxy_without_http_scheme( - self, proxy_mockserver: ProxyEchoMockServer - ) -> None: - with pytest.raises(UnsupportedURLSchemeError): - await maybe_deferred_to_future( - super().test_download_with_proxy_without_http_scheme(proxy_mockserver) # type: ignore[arg-type] - ) +@pytest.mark.skip(reason="Proxy support is not implemented yet") +@pytest.mark.requires_mitmproxy +class TestMitmProxy(H2DownloadHandlerMixin, TestMitmProxyBase): + pass + + +@pytest.mark.requires_internet +class TestRealWebsite(H2DownloadHandlerMixin, TestRealWebsiteBase): + @property + def platform_cert_store_works(self) -> bool: + return sys.platform != "win32" diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a8e635707..e685f607a 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -156,12 +156,12 @@ class HttpDownloadHandlerMock: @pytest.mark.requires_botocore class TestS3Anon: def setup_method(self): - crawler = get_crawler() - with mock.patch( - "scrapy.core.downloader.handlers.s3.HTTP11DownloadHandler", - HttpDownloadHandlerMock, - ): - self.s3reqh = build_from_crawler(S3DownloadHandler, crawler) + crawler = get_crawler( + settings_dict={ + "DOWNLOAD_HANDLERS": {"https": HttpDownloadHandlerMock}, + } + ) + self.s3reqh = build_from_crawler(S3DownloadHandler, crawler) self.download_request = self.s3reqh.download_request @coroutine_test @@ -183,28 +183,22 @@ class TestS3: settings_dict={ "AWS_ACCESS_KEY_ID": "0PN5J17HBGZHT7JJ3X82", "AWS_SECRET_ACCESS_KEY": "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", + "DOWNLOAD_HANDLERS": {"https": HttpDownloadHandlerMock}, } ) - with mock.patch( - "scrapy.core.downloader.handlers.s3.HTTP11DownloadHandler", - HttpDownloadHandlerMock, - ): - s3reqh = build_from_crawler(S3DownloadHandler, crawler) + s3reqh = build_from_crawler(S3DownloadHandler, crawler) self.download_request = s3reqh.download_request @contextlib.contextmanager def _mocked_date(self, date): - try: - import botocore.auth # noqa: F401,PLC0415 - except ImportError: + import botocore.auth # noqa: F401,PLC0415 + + # We need to mock botocore.auth.formatdate, because otherwise + # botocore overrides Date header with current date and time + # and Authorization header is different each time + with mock.patch("botocore.auth.formatdate") as mock_formatdate: + mock_formatdate.return_value = date yield - else: - # We need to mock botocore.auth.formatdate, because otherwise - # botocore overrides Date header with current date and time - # and Authorization header is different each time - with mock.patch("botocore.auth.formatdate") as mock_formatdate: - mock_formatdate.return_value = date - yield @coroutine_test async def test_request_signing1(self): diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 8e9fa0ae1..c4a8193c8 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -4,6 +4,8 @@ from __future__ import annotations import gzip import json +import logging +import os import re import sys from abc import ABC, abstractmethod @@ -11,10 +13,11 @@ from contextlib import asynccontextmanager from http import HTTPStatus from ipaddress import IPv4Address from socket import gethostbyname -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from urllib.parse import urlparse import pytest +from cryptography.x509 import load_der_x509_certificate from twisted.internet.ssl import Certificate from twisted.python.failure import Failure @@ -25,15 +28,21 @@ from scrapy.exceptions import ( DownloadFailedError, DownloadTimeoutError, ResponseDataLossError, + ScrapyDeprecationWarning, StopDownload, UnsupportedURLSchemeError, ) from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse +from scrapy.utils._deps_compat import ( + PYOPENSSL_X509_DEPRECATED, + TWISTED_TLS_LIMITS_OFFBY1, +) from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE +from tests.mockserver.mitm_proxy import wrong_credentials from tests.mockserver.proxy_echo import ProxyEchoMockServer from tests.mockserver.simple_https import SimpleMockServer from tests.spiders import ( @@ -41,6 +50,7 @@ from tests.spiders import ( BytesReceivedErrbackSpider, HeadersReceivedCallbackSpider, HeadersReceivedErrbackSpider, + SimpleSpider, SingleRequestSpider, ) from tests.utils.decorators import coroutine_test @@ -53,7 +63,20 @@ if TYPE_CHECKING: class TestHttpBase(ABC): - is_secure = False + is_secure: bool = False + http2: bool = False + # whether the handler supports per-request bindaddress + handler_supports_bindaddress_meta: bool = True + # RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a + # stream error (of type PROTOCOL_ERROR) so the client will send + # RST_STREAM. Some libraries do only this while e.g. h2 also closes the + # connection (see handling of ProtocolError in + # h2.connection.H2Connection.receive_data()), thus closing all streams that + # were using it, and we handle this as a normal exception. + handler_supports_http2_dataloss: bool = True + # default headers added by the underlying library that cannot be suppressed + always_present_req_headers: ClassVar[frozenset[str]] = frozenset() + default_handler_settings: ClassVar[dict[str, Any]] = {} @property @abstractmethod @@ -64,6 +87,10 @@ class TestHttpBase(ABC): async def get_dh( self, settings_dict: dict[str, Any] | None = None ) -> AsyncGenerator[DownloadHandlerProtocol]: + settings_dict = { + **self.default_handler_settings, + **(settings_dict or {}), + } crawler = get_crawler(DefaultSpider, settings_dict) crawler.spider = crawler._create_spider() dh = build_from_crawler(self.download_handler_cls, crawler) @@ -74,7 +101,7 @@ class TestHttpBase(ABC): @coroutine_test async def test_unsupported_scheme(self) -> None: - request = Request("ftp://unsupported.scheme") + request = Request("unsupp://unsupported.scheme") async with self.get_dh() as download_handler: with pytest.raises(UnsupportedURLSchemeError): await download_handler.download_request(request) @@ -183,6 +210,26 @@ class TestHttpBase(ABC): assert "headers" in body assert body["headers"]["X-Custom-Header"] == ["foo", "bar"] + @coroutine_test + async def test_server_receives_no_extra_headers( + self, mockserver: MockServer + ) -> None: + """Test that the handler doesn't add headers to the request.""" + request = Request(mockserver.url("/echo", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == HTTPStatus.OK + body = json.loads(response.body.decode("utf-8")) + assert "headers" in body + received_headers = set(body["headers"].keys()) + allowed_headers = { + "Connection", + "Content-Length", + "Host", + } | self.always_present_req_headers + extra_headers = received_headers - allowed_headers + assert not extra_headers, body["headers"] + @coroutine_test async def test_server_receives_correct_request_body( self, mockserver: MockServer @@ -235,9 +282,32 @@ class TestHttpBase(ABC): assert header_name in response.headers, ( f"Response was missing expected header {header_name}" ) - assert response.headers[header_name] == bytes( - header_value, encoding="utf-8" - ) + assert response.headers.getlist(header_name) == [ + header_value.encode(encoding="utf-8") + ] + + @coroutine_test + async def test_download_no_extra_response_headers( + self, mockserver: MockServer + ) -> None: + """Test that the handler doesn't add headers to the response.""" + request = Request( + mockserver.url("/response-headers", is_secure=self.is_secure), + headers={"content-type": "application/json"}, + body=json.dumps({}), + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + received_headers = set(response.headers.keys()) + allowed_headers = { + b"Content-Length", + b"Content-Type", + b"Date", + b"Server", + } + extra_headers = received_headers - allowed_headers + assert not extra_headers, response.headers @coroutine_test async def test_redirect_status(self, mockserver: MockServer) -> None: @@ -245,6 +315,7 @@ class TestHttpBase(ABC): async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.status == 302 + assert response.headers["Location"] == b"/redirected" @coroutine_test async def test_redirect_status_head(self, mockserver: MockServer) -> None: @@ -254,6 +325,7 @@ class TestHttpBase(ABC): async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.status == 302 + assert response.headers["Location"] == b"/redirected" @coroutine_test async def test_timeout_download_from_spider_nodata_rcvd( @@ -275,9 +347,7 @@ class TestHttpBase(ABC): @coroutine_test async def test_timeout_download_from_spider_server_hangs( - self, - mockserver: MockServer, - reactor_pytest: str, + self, mockserver: MockServer, reactor_pytest: str ) -> None: if reactor_pytest == "asyncio" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 @@ -359,9 +429,7 @@ class TestHttpBase(ABC): @coroutine_test async def test_response_header_content_length(self, mockserver: MockServer) -> None: - request = Request( - mockserver.url("/text", is_secure=self.is_secure), method="GET" - ) + request = Request(mockserver.url("/text", is_secure=self.is_secure)) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.headers[b"content-length"] == b"5" @@ -456,9 +524,19 @@ class TestHttpBase(ABC): assert "Cookie" not in headers assert "cookie" not in headers - -class TestHttp11Base(TestHttpBase): - """HTTP 1.1 test case""" + @coroutine_test + async def test_download_latency(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + await download_handler.download_request(request) + assert "download_latency" in request.meta + latency = request.meta["download_latency"] + if sys.version_info < (3, 13) and sys.platform == "win32": + # time.monotonic() resolution is too low here: + # https://docs.python.org/3/whatsnew/3.13.html#time + assert latency >= 0 + else: + assert latency > 0 @coroutine_test async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None: @@ -488,7 +566,7 @@ class TestHttp11Base(TestHttpBase): ) -> None: request = Request(mockserver.url("/text", is_secure=self.is_secure)) - # 10 is minimal size for this request and the limit is only counted on + # 5 is minimal size for this request and the limit is only counted on # response body. (regardless of headers) async with self.get_dh({"DOWNLOAD_MAXSIZE": 5}) as download_handler: response = await download_handler.download_request(request) @@ -509,9 +587,9 @@ class TestHttp11Base(TestHttpBase): async with self.get_dh({"DOWNLOAD_MAXSIZE": 1_500}) as download_handler: with pytest.raises(DownloadCancelledError): await download_handler.download_request(request) - assert ( - "Received 2048 bytes which is larger than download max size (1500)" - in caplog.text + assert re.search( + r"Received \d+ bytes which is larger than download max size \(1500\)", + caplog.text, ) @coroutine_test @@ -575,12 +653,11 @@ class TestHttp11Base(TestHttpBase): response = await download_handler.download_request(request) assert response.body == b"chunked content\n" - @pytest.mark.parametrize("url", ["broken", "broken-chunked"]) @coroutine_test - async def test_download_cause_data_loss( - self, url: str, mockserver: MockServer - ) -> None: - request = Request(mockserver.url(f"/{url}", is_secure=self.is_secure)) + async def test_download_cause_data_loss(self, mockserver: MockServer) -> None: + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) async with self.get_dh() as download_handler: with pytest.raises(ResponseDataLossError): await download_handler.download_request(request) @@ -589,6 +666,8 @@ class TestHttp11Base(TestHttpBase): async def test_download_cause_data_loss_double_warning( self, caplog: pytest.LogCaptureFixture, mockserver: MockServer ) -> None: + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") request = Request(mockserver.url("/broken", is_secure=self.is_secure)) async with self.get_dh() as download_handler: with pytest.raises(ResponseDataLossError): @@ -600,25 +679,43 @@ class TestHttp11Base(TestHttpBase): # no repeated warning assert "Got data loss" not in caplog.text - @pytest.mark.parametrize("url", ["broken", "broken-chunked"]) @coroutine_test - async def test_download_allow_data_loss( - self, url: str, mockserver: MockServer + async def test_download_allow_data_loss_broken( + self, mockserver: MockServer ) -> None: + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") request = Request( - mockserver.url(f"/{url}", is_secure=self.is_secure), + mockserver.url("/broken", is_secure=self.is_secure), meta={"download_fail_on_dataloss": False}, ) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.flags == ["dataloss"] + assert response.text == "partial" + + @coroutine_test + async def test_download_allow_data_loss_broken_chunked( + self, mockserver: MockServer + ) -> None: + if self.http2: + pytest.skip("Chunked encoding is specific to HTTP/1.1") + request = Request( + mockserver.url("/broken-chunked", is_secure=self.is_secure), + meta={"download_fail_on_dataloss": False}, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.flags == ["dataloss"] + assert response.text == "chunked content\n" - @pytest.mark.parametrize("url", ["broken", "broken-chunked"]) @coroutine_test async def test_download_allow_data_loss_via_setting( - self, url: str, mockserver: MockServer + self, mockserver: MockServer ) -> None: - request = Request(mockserver.url(f"/{url}", is_secure=self.is_secure)) + if self.http2 and not self.handler_supports_http2_dataloss: + pytest.skip("This handler doesn't support dataloss on HTTP/2") + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) async with self.get_dh( {"DOWNLOAD_FAIL_ON_DATALOSS": False} ) as download_handler: @@ -645,6 +742,12 @@ class TestHttp11Base(TestHttpBase): @coroutine_test async def test_download_conn_aborted(self, mockserver: MockServer) -> None: # copy of TestCrawl.test_retry_conn_aborted() + if self.http2: + # it may be possible to write a separate resource that does something + # suitable on HTTP/2 without sending Content-Length + pytest.skip( + "On HTTP/2 this triggers a Content-Length mismatch error instead." + ) request = Request(mockserver.url("/drop?abort=1", is_secure=self.is_secure)) async with self.get_dh() as download_handler: with pytest.raises(DownloadFailedError): @@ -664,20 +767,82 @@ class TestHttp11Base(TestHttpBase): @coroutine_test async def test_protocol(self, mockserver: MockServer) -> None: - request = Request( - mockserver.url("/host", is_secure=self.is_secure), method="GET" - ) + request = Request(mockserver.url("/host", is_secure=self.is_secure)) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) assert response.protocol == "HTTP/1.1" + @pytest.mark.skipif( + sys.platform == "darwin", + reason="127.0.0.2 is not available on macOS by default", + ) + @pytest.mark.parametrize("setting_value", [("127.0.0.2", 0), "127.0.0.2"]) + @coroutine_test + async def test_download_bind_address_setting( + self, mockserver: MockServer, setting_value: Any + ) -> None: + request = Request(mockserver.url("/client-ip", is_secure=self.is_secure)) + async with self.get_dh( + {"DOWNLOAD_BIND_ADDRESS": setting_value} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"127.0.0.2" -class TestHttps11Base(TestHttp11Base): + @pytest.mark.skipif( + sys.platform == "darwin", + reason="127.0.0.2 is not available on macOS by default", + ) + @pytest.mark.parametrize("meta_value", [("127.0.0.2", 0), "127.0.0.2"]) + @coroutine_test + async def test_download_bind_address_meta( + self, mockserver: MockServer, caplog: pytest.LogCaptureFixture, meta_value: Any + ) -> None: + request = Request( + mockserver.url("/client-ip", is_secure=self.is_secure), + meta={"bindaddress": meta_value}, + ) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + if self.handler_supports_bindaddress_meta: + assert response.body == b"127.0.0.2" + else: + assert ( + "The 'bindaddress' request meta key is not supported by" in caplog.text + ) + + @coroutine_test + async def test_verbatim_url(self, mockserver: MockServer) -> None: + # Square brackets are encoded by safe_url_string (w3lib). + path = "/uri/items?data[0]=a" + url = mockserver.url(path, is_secure=self.is_secure) + + # Without verbatim_url, the brackets are percent-encoded before the + # request reaches the server. + request = Request(url) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"/uri/items?data%5B0%5D=a" + + # With verbatim_url=True the URL is sent to the server as-is. + request = Request(url, meta={"verbatim_url": True}) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == path.encode() + + +class TestHttpsBase(TestHttpBase): is_secure = True tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' - 'subject "/C=IE/O=Scrapy/CN=localhost"' + ( + 'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", ' + 'subject "CN=localhost,O=Scrapy,C=IE"' + ) + if PYOPENSSL_X509_DEPRECATED + else ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' + 'subject "/C=IE/O=Scrapy/CN=localhost"' + ) ) def test_download_conn_lost(self) -> None: # type: ignore[override] @@ -699,6 +864,38 @@ class TestHttps11Base(TestHttp11Base): assert response.body == b"Works" assert self.tls_log_message in caplog.text + @coroutine_test + async def test_verify_certs_deprecated(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + with ( # noqa: PT031 + pytest.warns( + ScrapyDeprecationWarning, + match="'DOWNLOADER_CLIENTCONTEXTFACTORY' setting is deprecated", + ), + pytest.warns( + ScrapyDeprecationWarning, + match="BrowserLikeContextFactory is deprecated", + ), + ): + async with self.get_dh( + { + "DOWNLOADER_CLIENTCONTEXTFACTORY": "scrapy.core.downloader.contextfactory.BrowserLikeContextFactory" + } + ) as download_handler: + with pytest.raises( + (DownloadConnectionRefusedError, DownloadFailedError) + ): + await download_handler.download_request(request) + + @coroutine_test + async def test_verify_certs(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/text", is_secure=self.is_secure)) + async with self.get_dh( + {"DOWNLOAD_VERIFY_CERTIFICATES": True} + ) as download_handler: + with pytest.raises((DownloadConnectionRefusedError, DownloadFailedError)): + await download_handler.download_request(request) + class TestSimpleHttpsBase(ABC): """Base class for special cases tested with just one simple request""" @@ -709,16 +906,17 @@ class TestSimpleHttpsBase(ABC): cipher_string: str | None = None @pytest.fixture(scope="class") - def simple_mockserver(self) -> Generator[SimpleMockServer]: + @classmethod + def simple_mockserver(cls) -> Generator[SimpleMockServer]: with SimpleMockServer( - self.keyfile, self.certfile, self.cipher_string + cls.keyfile, cls.certfile, cipher_string=cls.cipher_string ) as simple_mockserver: yield simple_mockserver @pytest.fixture(scope="class") - def url(self, simple_mockserver: SimpleMockServer) -> str: - # need to use self.host instead of what mockserver returns - return f"https://{self.host}:{simple_mockserver.port(is_secure=True)}/file" + @classmethod + def url(cls, simple_mockserver: SimpleMockServer) -> str: + return f"https://{cls.host}:{simple_mockserver.port(is_secure=True)}/file" @property @abstractmethod @@ -774,6 +972,143 @@ class TestHttpsCustomCiphersBase(TestSimpleHttpsBase): cipher_string = "CAMELLIA256-SHA" +class TestHttpsTLSVersionBase(ABC): + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + @asynccontextmanager + async def get_dh( + self, client_tls_min: str | None, client_tls_max: str | None + ) -> AsyncGenerator[DownloadHandlerProtocol]: + settings = {} + if client_tls_min is not None: + settings["DOWNLOAD_TLS_MIN_VERSION"] = client_tls_min + if client_tls_max is not None: + settings["DOWNLOAD_TLS_MAX_VERSION"] = client_tls_max + crawler = get_crawler(DefaultSpider, settings_dict=settings) + crawler.spider = crawler._create_spider() + dh = build_from_crawler(self.download_handler_cls, crawler) + try: + yield dh + finally: + await dh.close() + + @pytest.mark.parametrize( + ( + "server_tls_min", + "server_tls_max", + "client_tls_min", + "client_tls_max", + "expect_success", + ), + [ + pytest.param(None, None, None, None, True, id="no-limits"), + pytest.param(None, None, None, "TLSv1.2", True, id="client-max-tls1.2"), + pytest.param(None, None, "TLSv1.3", None, True, id="client-min-tls1.3"), + pytest.param( + "TLSv1.3", + None, + None, + "TLSv1.2", + False, + id="client-max-below-server-min", + ), + pytest.param( + None, + "TLSv1.2", + "TLSv1.3", + None, + False, + id="client-min-above-server-max", + ), + pytest.param(None, "TLSv1.2", None, "TLSv1.2", True, id="both-tls1.2"), + pytest.param("TLSv1.3", None, "TLSv1.3", None, True, id="both-tls1.3"), + pytest.param( + None, + None, + "TLSv1.0", + None, + True, + id="client-min-tls1.0", + marks=pytest.mark.filterwarnings( + r"ignore:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning" + ), + ), + pytest.param( + "TLSv1.0", + None, + "TLSv1.0", + None, + True, + id="both-min-tls1.0", + marks=pytest.mark.filterwarnings( + r"ignore:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning" + ), + ), + pytest.param( + "TLSv1.2", + "TLSv1.3", + "TLSv1.2", + "TLSv1.3", + True, + id="both-tls1.2-1.3", + marks=pytest.mark.xfail( + TWISTED_TLS_LIMITS_OFFBY1, + reason="Can't set max to 1.3 on this Twisted version", + strict=True, + ), + ), + ], + ) + @coroutine_test + async def test_download( + self, + server_tls_min: str | None, + server_tls_max: str | None, + client_tls_min: str | None, + client_tls_max: str | None, + expect_success: bool, + ) -> None: + with SimpleMockServer( + self.keyfile, + self.certfile, + tls_min_version=server_tls_min, + tls_max_version=server_tls_max, + ) as simple_mockserver: + url = f"https://localhost:{simple_mockserver.port(is_secure=True)}/file" + request = Request(url) + async with self.get_dh(client_tls_min, client_tls_max) as dh: + if expect_success: + response = await dh.download_request(request) + assert response.body == b"0123456789" + else: + with pytest.raises( + (DownloadConnectionRefusedError, DownloadFailedError) + ): + await dh.download_request(request) + + @coroutine_test + async def test_invalid_min_version_setting(self) -> None: + with pytest.raises( + ValueError, match="Unknown DOWNLOAD_TLS_MIN_VERSION value: invalid" + ): + async with self.get_dh(client_tls_min="invalid", client_tls_max=None): + pass + + @coroutine_test + async def test_invalid_max_version_setting(self) -> None: + with pytest.raises( + ValueError, match="Unknown DOWNLOAD_TLS_MAX_VERSION value: invalid" + ): + async with self.get_dh(client_tls_min=None, client_tls_max="invalid"): + pass + + class TestHttpWithCrawlerBase(ABC): @property @abstractmethod @@ -813,20 +1148,27 @@ class TestHttpWithCrawlerBase(ABC): reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined] assert reason == "finished" + @pytest.mark.filterwarnings( + r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning" + ) @coroutine_test async def test_response_ssl_certificate(self, mockserver: MockServer) -> None: if not self.is_secure: pytest.skip("Only applies to HTTPS") # copy of TestCrawl.test_response_ssl_certificate() - # the current test implementation can only work for Twisted-based download handlers crawler = get_crawler(SingleRequestSpider, self.settings_dict) url = mockserver.url("/echo?body=test", is_secure=self.is_secure) await crawler.crawl_async(seed=url, mockserver=mockserver) assert isinstance(crawler.spider, SingleRequestSpider) cert = crawler.spider.meta["responses"][0].certificate - assert isinstance(cert, Certificate) - assert cert.getSubject().commonName == b"localhost" - assert cert.getIssuer().commonName == b"localhost" + assert cert is not None + if isinstance(cert, Certificate): # Twisted + assert cert.getSubject().commonName == b"localhost" + assert cert.getIssuer().commonName == b"localhost" + elif isinstance(cert, bytes): # DER bytes + cert_x509 = load_der_x509_certificate(cert) + assert cert_x509.subject.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" + assert cert_x509.issuer.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" @coroutine_test async def test_response_ip_address(self, mockserver: MockServer) -> None: @@ -912,6 +1254,8 @@ class TestHttpWithCrawlerBase(ABC): class TestHttpProxyBase(ABC): is_secure = False expected_http_proxy_request_body = b"http://example.com" + # whether the handler supports HTTPS proxies with HTTPS destinations + handler_supports_tls_in_tls: bool = True @property @abstractmethod @@ -964,6 +1308,8 @@ class TestHttpProxyBase(ABC): ) -> None: if NON_EXISTING_RESOLVABLE: pytest.skip("Non-existing hosts are resolvable") + if self.is_secure and not self.handler_supports_tls_in_tls: + pytest.skip("HTTPS proxies for HTTPS destinations are not supported") http_proxy = proxy_mockserver.url("", is_secure=self.is_secure) domain = "https://no-such-domain.nosuch" request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2}) @@ -983,3 +1329,233 @@ class TestHttpProxyBase(ABC): assert response.status == 200 assert response.url == request.url assert response.body == self.expected_http_proxy_request_body + + +PROXY_KINDS = ["http", "https", "socks5"] + + +class TestMitmProxyBase(ABC): + # whether the handler supports HTTPS proxies with HTTPS destinations + handler_supports_tls_in_tls: bool = True + handler_supports_socks: bool = False + + @property + @abstractmethod + def settings_dict(self) -> dict[str, Any] | None: + raise NotImplementedError + + def _maybe_skip(self, proxy_kind: str, https_dest: bool) -> None: + if proxy_kind == "socks5" and not self.handler_supports_socks: + pytest.skip("SOCKS proxies are not supported") + if ( + proxy_kind == "https" + and https_dest + and not self.handler_supports_tls_in_tls + ): + pytest.skip("HTTPS proxies for HTTPS destinations are not supported") + + @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True) + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @coroutine_test + async def test_proxy( + self, + caplog: pytest.LogCaptureFixture, + proxy_server: str, + mockserver: MockServer, + https_dest: bool, + ) -> None: + """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination.""" + self._maybe_skip(proxy_server, https_dest) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=200", is_secure=https_dest) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) + + @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True) + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @coroutine_test + async def test_proxy_auth_error( + self, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + proxy_server: str, + mockserver: MockServer, + https_dest: bool, + ) -> None: + """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination, wrong proxy creds.""" + self._maybe_skip(proxy_server, https_dest) + envvar = "https_proxy" if https_dest else "http_proxy" + monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar])) + crawler = get_crawler(SimpleSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/status?n=200", is_secure=https_dest) + ) + if proxy_server == "socks5": + assert "DownloadConnectionRefusedError" in caplog.text + else: + # The proxy returns a 407 error code but it does not reach the + # client; it just sees an exception. + self._assert_got_auth_exception(caplog.text) + + @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True) + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @coroutine_test + async def test_proxy_dont_leak_auth_header( + self, + caplog: pytest.LogCaptureFixture, + proxy_server: str, + mockserver: MockServer, + https_dest: bool, + ) -> None: + """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination. Check that the + auth header is not sent to the destination.""" + self._maybe_skip(proxy_server, https_dest) + request = Request(mockserver.url("/echo", is_secure=https_dest)) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async(seed=request) + assert isinstance(crawler.spider, SingleRequestSpider) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) + echo = json.loads(crawler.spider.meta["responses"][0].text) + assert "Proxy-Authorization" not in echo["headers"] + + @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True) + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @coroutine_test + async def test_proxy_redirect( + self, + caplog: pytest.LogCaptureFixture, + proxy_server: str, + mockserver: MockServer, + https_dest: bool, + ) -> None: + """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination, following a + redirect. Check that the redirected request still goes through the + proxy and doesn't lose the proxy auth. + """ + self._maybe_skip(proxy_server, https_dest) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/redirect", is_secure=https_dest) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + assert crawler.spider.meta.get("failure") is None + responses = crawler.spider.meta.get("responses", []) + assert len(responses) == 1 + assert responses[0].status == 200 + assert responses[0].url == mockserver.url("/redirected", is_secure=https_dest) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(responses[0].headers, https_dest) + + @staticmethod + def _assert_headers(headers: Headers, https_dest: bool) -> None: + assert b"X-Via-Mitmproxy" in headers + if https_dest: + assert b"X-Via-Mitmproxy-TLS" in headers + + @staticmethod + def _assert_got_response_code(code: int, log: str) -> None: + assert str(log).count(f"Crawled ({code})") == 1 + + @staticmethod + def _assert_got_auth_exception(log: str) -> None: + assert "Proxy Authentication Required" in log or "407" in log + + +class TestRealWebsiteBase(ABC): + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + @property + @abstractmethod + def settings_dict(self) -> dict[str, Any] | None: + raise NotImplementedError + + @property + def platform_cert_store_works(self) -> bool: + """Whether valid certificates can be verified. + + Twisted on Windows cannot do that out of the box, see e.g. + https://github.com/twisted/twisted/issues/6371. + """ + return True + + @asynccontextmanager + async def get_dh( + self, settings_dict: dict[str, Any] | None = None + ) -> AsyncGenerator[DownloadHandlerProtocol]: + crawler = get_crawler(DefaultSpider, settings_dict) + crawler.spider = crawler._create_spider() + dh = build_from_crawler(self.download_handler_cls, crawler) + try: + yield dh + finally: + await dh.close() + + @coroutine_test + async def test_download(self) -> None: + request = Request("https://books.toscrape.com/") + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + assert "All products | Books to Scrape - Sandbox" in response.text + + @coroutine_test + async def test_download_with_spider(self) -> None: + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + await maybe_deferred_to_future( + crawler.crawl(seed=Request("https://books.toscrape.com/")) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + failure = crawler.spider.meta.get("failure") + assert failure is None + reason = crawler.spider.meta["close_reason"] + assert reason == "finished" + + @coroutine_test + async def test_verify_certs(self) -> None: + if not self.platform_cert_store_works: + pytest.skip("Cannot verify certificates") + request = Request("https://books.toscrape.com/") + async with self.get_dh( + {"DOWNLOAD_VERIFY_CERTIFICATES": True} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + assert "All products | Books to Scrape - Sandbox" in response.text + + @pytest.mark.parametrize("verify_certs", [True, False]) + @coroutine_test + async def test_tls_logging( + self, caplog: pytest.LogCaptureFixture, verify_certs: bool + ) -> None: + if verify_certs and not self.platform_cert_store_works: + pytest.skip("Cannot verify certificates") + request = Request("https://books.toscrape.com/") + async with self.get_dh( + { + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True, + "DOWNLOAD_VERIFY_CERTIFICATES": verify_certs, + } + ) as download_handler: + with caplog.at_level("DEBUG"): + response = await download_handler.download_request(request) + assert response.status == 200 + assert "SSL connection to books.toscrape.com using protocol" in caplog.text diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py deleted file mode 100644 index 44084f1e8..000000000 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ /dev/null @@ -1,62 +0,0 @@ -import pytest - -from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware -from scrapy.http import HtmlResponse, Request, Response -from scrapy.spiders import Spider -from scrapy.utils.test import get_crawler - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestAjaxCrawlMiddleware: - def setup_method(self): - crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True}) - self.spider = crawler._create_spider("foo") - self.mw = AjaxCrawlMiddleware.from_crawler(crawler) - - def _ajaxcrawlable_body(self): - return b'' - - def _req_resp(self, url, req_kwargs=None, resp_kwargs=None): - req = Request(url, **(req_kwargs or {})) - resp = HtmlResponse(url, request=req, **(resp_kwargs or {})) - return req, resp - - def test_non_get(self): - req, resp = self._req_resp("http://example.com/", {"method": "HEAD"}) - resp2 = self.mw.process_response(req, resp, self.spider) - assert resp == resp2 - - def test_binary_response(self): - req = Request("http://example.com/") - resp = Response("http://example.com/", body=b"foobar\x00\x01\x02", request=req) - resp2 = self.mw.process_response(req, resp, self.spider) - assert resp is resp2 - - def test_ajaxcrawl(self): - req, resp = self._req_resp( - "http://example.com/", - {"meta": {"foo": "bar"}}, - {"body": self._ajaxcrawlable_body()}, - ) - req2 = self.mw.process_response(req, resp, self.spider) - assert req2.url == "http://example.com/?_escaped_fragment_=" - assert req2.meta["foo"] == "bar" - - def test_ajaxcrawl_loop(self): - req, resp = self._req_resp( - "http://example.com/", {}, {"body": self._ajaxcrawlable_body()} - ) - req2 = self.mw.process_response(req, resp, self.spider) - resp2 = HtmlResponse(req2.url, body=resp.body, request=req2) - resp3 = self.mw.process_response(req2, resp2, self.spider) - - assert isinstance(resp3, HtmlResponse), (resp3.__class__, resp3) - assert resp3.request.url == "http://example.com/?_escaped_fragment_=" - assert resp3 is resp2 - - def test_noncrawlable_body(self): - req, resp = self._req_resp( - "http://example.com/", {}, {"body": b""} - ) - resp2 = self.mw.process_response(req, resp, self.spider) - assert resp is resp2 diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 49215329e..225562644 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -42,7 +42,7 @@ def _cookie_to_set_cookie_value(cookie): def _cookies_to_set_cookie_list(cookies): """Given a group of cookie defined either as a dictionary or as a list of dictionaries (i.e. in a format supported by the cookies parameter of - Request), return the equivalen list of strings that can be associated to a + Request), return the equivalent list of strings that can be associated to a ``Set-Cookie`` header.""" if not cookies: return [] diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 522a3002f..827133d2e 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -2,8 +2,25 @@ import pytest from w3lib.http import basic_auth_header from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler + +_DOMAIN_NOT_SET = object() + + +def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET): + settings: dict = { + "HTTPAUTH_USER": user, + "HTTPAUTH_PASS": passwd, + } + if domain is not _DOMAIN_NOT_SET: + settings["HTTPAUTH_DOMAIN"] = domain + return HttpAuthMiddleware.from_crawler(get_crawler(settings_dict=settings)) + + +# --- Spider attribute tests (deprecated) --- class LegacySpider(Spider): @@ -23,61 +40,135 @@ class AnyDomainSpider(Spider): http_auth_domain = None -class TestHttpAuthMiddlewareLegacy: - def setup_method(self): - self.spider = LegacySpider("foo") - - def test_auth(self): +class TestHttpAuthMiddlewareLegacySpiderAttr: + def test_missing_domain_raises(self): mw = HttpAuthMiddleware() - with pytest.raises(AttributeError): - mw.spider_opened(self.spider) + with pytest.warns(ScrapyDeprecationWarning), pytest.raises(AttributeError): + mw.spider_opened(LegacySpider("foo")) + def test_domain_spider(self): + mw = HttpAuthMiddleware() + with pytest.warns(ScrapyDeprecationWarning): + mw.spider_opened(DomainSpider("foo")) + req = Request("http://example.com/") + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") -class TestHttpAuthMiddleware: - def setup_method(self): - self.mw = HttpAuthMiddleware() - spider = DomainSpider("foo") - self.mw.spider_opened(spider) - - def teardown_method(self): - del self.mw - - def test_no_auth(self): - req = Request("http://example-noauth.com/") - assert self.mw.process_request(req) is None + def test_no_auth_wrong_domain(self): + mw = HttpAuthMiddleware() + with pytest.warns(ScrapyDeprecationWarning): + mw.spider_opened(DomainSpider("foo")) + req = Request("http://other.com/") + mw.process_request(req) assert "Authorization" not in req.headers - def test_auth_domain(self): + def test_any_domain_spider(self): + mw = HttpAuthMiddleware() + with pytest.warns(ScrapyDeprecationWarning): + mw.spider_opened(AnyDomainSpider("foo")) + req = Request("http://anywhere.com/") + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") + + +# --- Settings-based tests --- + + +class TestHttpAuthMiddlewareSettings: + def test_no_auth(self): + mw = make_mw() req = Request("http://example.com/") - assert self.mw.process_request(req) is None + mw.process_request(req) + assert "Authorization" not in req.headers + + def test_auth_without_domain_raises(self): + with pytest.raises(ValueError, match="HTTPAUTH_DOMAIN"): + make_mw(user="foo", passwd="bar") + + def test_auth_all_domains(self): + mw = make_mw(user="foo", passwd="bar", domain=None) + req = Request("http://example.com/") + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header("foo", "bar") + + def test_auth_domain_match(self): + mw = make_mw(user="foo", passwd="bar", domain="example.com") + req = Request("http://example.com/") + mw.process_request(req) assert req.headers["Authorization"] == basic_auth_header("foo", "bar") def test_auth_subdomain(self): - req = Request("http://foo.example.com/") - assert self.mw.process_request(req) is None + mw = make_mw(user="foo", passwd="bar", domain="example.com") + req = Request("http://sub.example.com/") + mw.process_request(req) assert req.headers["Authorization"] == basic_auth_header("foo", "bar") + def test_no_auth_wrong_domain(self): + mw = make_mw(user="foo", passwd="bar", domain="example.com") + req = Request("http://other.com/") + mw.process_request(req) + assert "Authorization" not in req.headers + def test_auth_already_set(self): + mw = make_mw(user="foo", passwd="bar", domain="example.com") req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) - assert self.mw.process_request(req) is None + mw.process_request(req) assert req.headers["Authorization"] == b"Digest 123" -class TestHttpAuthAnyMiddleware: - def setup_method(self): - self.mw = HttpAuthMiddleware() - spider = AnyDomainSpider("foo") - self.mw.spider_opened(spider) +# --- Per-request meta tests --- - def teardown_method(self): - del self.mw - def test_auth(self): - req = Request("http://example.com/") - assert self.mw.process_request(req) is None - assert req.headers["Authorization"] == basic_auth_header("foo", "bar") +class TestHttpAuthMiddlewareMeta: + def test_meta_auth_no_domain(self): + mw = make_mw() + req = Request("http://example.com/", meta={"http_user": "u", "http_pass": "p"}) + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header("u", "p") - def test_auth_already_set(self): - req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) - assert self.mw.process_request(req) is None + def test_meta_auth_domain_match(self): + mw = make_mw() + req = Request( + "http://example.com/", + meta={ + "http_user": "u", + "http_pass": "p", + "http_auth_domain": "example.com", + }, + ) + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header("u", "p") + + def test_meta_auth_domain_no_match(self): + mw = make_mw() + req = Request( + "http://other.com/", + meta={ + "http_user": "u", + "http_pass": "p", + "http_auth_domain": "example.com", + }, + ) + mw.process_request(req) + assert "Authorization" not in req.headers + + def test_meta_overrides_middleware(self): + mw = make_mw(user="mw_user", passwd="mw_pass", domain="example.com") + req = Request( + "http://example.com/", + meta={"http_user": "meta_user", "http_pass": "meta_pass"}, + ) + mw.process_request(req) + assert req.headers["Authorization"] == basic_auth_header( + "meta_user", "meta_pass" + ) + + def test_meta_already_set(self): + mw = make_mw() + req = Request( + "http://example.com/", + headers={"Authorization": "Digest 123"}, + meta={"http_user": "u", "http_pass": "p"}, + ) + mw.process_request(req) assert req.headers["Authorization"] == b"Digest 123" diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index e94591038..e5d726764 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -89,20 +89,6 @@ class TestBase: assert response1.headers == response2.headers assert response1.body == response2.body - def assertEqualRequest(self, request1, request2): - assert request1.url == request2.url - assert request1.headers == request2.headers - assert request1.body == request2.body - - def assertEqualRequestButWithCacheValidators(self, request1, request2): - assert request1.url == request2.url - assert b"If-None-Match" not in request1.headers - assert b"If-Modified-Since" not in request1.headers - assert any( - h in request2.headers for h in (b"If-None-Match", b"If-Modified-Since") - ) - assert request1.body == request2.body - class StorageTestMixin: """Mixin containing storage-specific test methods.""" @@ -217,7 +203,7 @@ class DummyPolicyTestMixin(PolicyTestMixin): assert mw.process_request(req) is None # s3 scheme response is cached by default - req, res = Request("s3://bucket/key"), Response("http://bucket/key") + req, res = Request("s3://bucket/key"), Response("s3://bucket/key") with self._middleware() as mw: assert mw.process_request(req) is None mw.process_response(req, res) @@ -228,7 +214,7 @@ class DummyPolicyTestMixin(PolicyTestMixin): assert "cached" in cached.flags # ignore s3 scheme - req, res = Request("s3://bucket/key2"), Response("http://bucket/key2") + req, res = Request("s3://bucket/key2"), Response("s3://bucket/key2") with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw: assert mw.process_request(req) is None mw.process_response(req, res) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index bb7fcd6c7..30caa094f 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -361,7 +361,7 @@ class TestHttpCompression: zf.write(plainbody) zf.close() response = Response( - "http;//www.example.com/", headers=headers, body=f.getvalue() + "http://www.example.com/", headers=headers, body=f.getvalue() ) request = Request("http://www.example.com/") @@ -386,7 +386,7 @@ class TestHttpCompression: zf.write(plainbody) zf.close() response = HtmlResponse( - "http;//www.example.com/page.html", headers=headers, body=f.getvalue() + "http://www.example.com/page.html", headers=headers, body=f.getvalue() ) request = Request("http://www.example.com/") @@ -493,7 +493,7 @@ class TestHttpCompression: gz_resp.close() response = Response( - "http;//www.example.com/", headers=headers, body=r.getvalue() + "http://www.example.com/", headers=headers, body=r.getvalue() ) request = Request("http://www.example.com/") diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index dc0a31a76..edfb15d10 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -1,5 +1,3 @@ -import warnings - import pytest from scrapy import Request, Spider @@ -120,9 +118,7 @@ def test_process_request_invalid_domains(): allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) mw = OffsiteMiddleware.from_crawler(crawler) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - mw.spider_opened(crawler.spider) + mw.spider_opened(crawler.spider) request = Request("https://a.example") assert mw.process_request(request) is None for letter in ("b", "c"): @@ -210,9 +206,7 @@ def test_request_scheduled_invalid_domains(): allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) mw = OffsiteMiddleware.from_crawler(crawler) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - mw.spider_opened(crawler.spider) + mw.spider_opened(crawler.spider) request = Request("https://a.example") assert mw.request_scheduled(request, crawler.spider) is None for letter in ("b", "c"): diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 281bc898a..42a25cd5b 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,993 +1,25 @@ -from itertools import chain, product +import logging +from unittest.mock import MagicMock import pytest -from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware -from scrapy.downloadermiddlewares.redirect import ( - MetaRefreshMiddleware, - RedirectMiddleware, +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware +from scrapy.http import Request, Response +from scrapy.spidermiddlewares.referer import ( + POLICY_NO_REFERRER, + POLICY_ORIGIN, + POLICY_UNSAFE_URL, + RefererMiddleware, ) -from scrapy.exceptions import IgnoreRequest -from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider -from scrapy.utils.misc import set_environ +from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler - - -class Base: - class Test: - def test_priority_adjust(self): - req = Request("http://a.com") - rsp = self.get_response(req, "http://a.com/redirected") - req2 = self.mw.process_response(req, rsp) - assert req2.priority > req.priority - - def test_dont_redirect(self): - url = "http://www.example.com/301" - url2 = "http://www.example.com/redirected" - req = Request(url, meta={"dont_redirect": True}) - rsp = self.get_response(req, url2) - - r = self.mw.process_response(req, rsp) - assert isinstance(r, Response) - assert r is rsp - - # Test that it redirects when dont_redirect is False - req = Request(url, meta={"dont_redirect": False}) - rsp = self.get_response(req, url2) - - r = self.mw.process_response(req, rsp) - assert isinstance(r, Request) - - def test_post(self): - url = "http://www.example.com/302" - url2 = "http://www.example.com/redirected2" - req = Request( - url, - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = self.get_response(req, url2) - - req2 = self.mw.process_response(req, rsp) - assert isinstance(req2, Request) - assert req2.url == url2 - assert req2.method == "GET" - assert "Content-Type" not in req2.headers, ( - "Content-Type header must not be present in redirected request" - ) - assert "Content-Length" not in req2.headers, ( - "Content-Length header must not be present in redirected request" - ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - def test_max_redirect_times(self): - self.mw.max_redirect_times = 1 - req = Request("http://scrapytest.org/302") - rsp = self.get_response(req, "/redirected") - - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert "redirect_times" in req.meta - assert req.meta["redirect_times"] == 1 - with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp) - - def test_ttl(self): - self.mw.max_redirect_times = 100 - req = Request("http://scrapytest.org/302", meta={"redirect_ttl": 1}) - rsp = self.get_response(req, "/a") - - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - with pytest.raises(IgnoreRequest): - self.mw.process_response(req, rsp) - - def test_redirect_urls(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = self.get_response(req1, "/redirected") - req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req1, "/redirected2") - req3 = self.mw.process_response(req2, rsp2) - - assert req2.url == "http://scrapytest.org/redirected" - assert req2.meta["redirect_urls"] == ["http://scrapytest.org/first"] - assert req3.url == "http://scrapytest.org/redirected2" - assert req3.meta["redirect_urls"] == [ - "http://scrapytest.org/first", - "http://scrapytest.org/redirected", - ] - - def test_redirect_reasons(self): - req1 = Request("http://scrapytest.org/first") - rsp1 = self.get_response(req1, "/redirected1") - req2 = self.mw.process_response(req1, rsp1) - rsp2 = self.get_response(req2, "/redirected2") - req3 = self.mw.process_response(req2, rsp2) - assert req2.meta["redirect_reasons"] == [self.reason] - assert req3.meta["redirect_reasons"] == [self.reason, self.reason] - - def test_cross_origin_header_dropping(self): - safe_headers = {"A": "B"} - cookie_header = {"Cookie": "a=b"} - authorization_header = {"Authorization": "Bearer 123456"} - - original_request = Request( - "https://example.com", - headers={**safe_headers, **cookie_header, **authorization_header}, - ) - - # Redirects to the same origin (same scheme, same domain, same port) - # keep all headers. - internal_response = self.get_response( - original_request, "https://example.com/a" - ) - internal_redirect_request = self.mw.process_response( - original_request, internal_response - ) - assert isinstance(internal_redirect_request, Request) - assert original_request.headers == internal_redirect_request.headers - - # Redirects to the same origin (same scheme, same domain, same port) - # keep all headers also when the scheme is http. - http_request = Request( - "http://example.com", - headers={**safe_headers, **cookie_header, **authorization_header}, - ) - http_response = self.get_response(http_request, "http://example.com/a") - http_redirect_request = self.mw.process_response( - http_request, http_response - ) - assert isinstance(http_redirect_request, Request) - assert http_request.headers == http_redirect_request.headers - - # For default ports, whether the port is explicit or implicit does not - # affect the outcome, it is still the same origin. - to_explicit_port_response = self.get_response( - original_request, "https://example.com:443/a" - ) - to_explicit_port_redirect_request = self.mw.process_response( - original_request, to_explicit_port_response - ) - assert isinstance(to_explicit_port_redirect_request, Request) - assert original_request.headers == to_explicit_port_redirect_request.headers - - # For default ports, whether the port is explicit or implicit does not - # affect the outcome, it is still the same origin. - to_implicit_port_response = self.get_response( - original_request, "https://example.com/a" - ) - to_implicit_port_redirect_request = self.mw.process_response( - original_request, to_implicit_port_response - ) - assert isinstance(to_implicit_port_redirect_request, Request) - assert original_request.headers == to_implicit_port_redirect_request.headers - - # A port change drops the Authorization header because the origin - # changes, but keeps the Cookie header because the domain remains the - # same. - different_port_response = self.get_response( - original_request, "https://example.com:8080/a" - ) - different_port_redirect_request = self.mw.process_response( - original_request, different_port_response - ) - assert isinstance(different_port_redirect_request, Request) - assert { - **safe_headers, - **cookie_header, - } == different_port_redirect_request.headers.to_unicode_dict() - - # A domain change drops both the Authorization and the Cookie header. - external_response = self.get_response( - original_request, "https://example.org/a" - ) - external_redirect_request = self.mw.process_response( - original_request, external_response - ) - assert isinstance(external_redirect_request, Request) - assert safe_headers == external_redirect_request.headers.to_unicode_dict() - - # A scheme upgrade (http → https) drops the Authorization header - # because the origin changes, but keeps the Cookie header because the - # domain remains the same. - upgrade_response = self.get_response(http_request, "https://example.com/a") - upgrade_redirect_request = self.mw.process_response( - http_request, upgrade_response - ) - assert isinstance(upgrade_redirect_request, Request) - assert { - **safe_headers, - **cookie_header, - } == upgrade_redirect_request.headers.to_unicode_dict() - - # A scheme downgrade (https → http) drops the Authorization header - # because the origin changes, and the Cookie header because its value - # cannot indicate whether the cookies were secure (HTTPS-only) or not. - # - # Note: If the Cookie header is set by the cookie management - # middleware, as recommended in the docs, the dropping of Cookie on - # scheme downgrade is not an issue, because the cookie management - # middleware will add again the Cookie header to the new request if - # appropriate. - downgrade_response = self.get_response( - original_request, "http://example.com/a" - ) - downgrade_redirect_request = self.mw.process_response( - original_request, downgrade_response - ) - assert isinstance(downgrade_redirect_request, Request) - assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() - - def test_meta_proxy_http_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_http_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_http_to_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("http://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_meta_proxy_https_to_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - meta = {"proxy": "https://a:@a.example"} - request1 = Request("https://example.com", meta=meta) - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_http_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_http_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_https_absolute(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_https_relative(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "/a") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "/a") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_proxied_http_to_proxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request2.meta["_auth_proxy"] == "https://b.example" - assert request2.meta["proxy"] == "https://b.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_proxied_http_to_unproxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request1.meta["_auth_proxy"] == "https://a.example" - assert request1.meta["proxy"] == "https://a.example" - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request3.meta["_auth_proxy"] == "https://a.example" - assert request3.meta["proxy"] == "https://a.example" - - def test_system_proxy_unproxied_http_to_proxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request2.meta["_auth_proxy"] == "https://b.example" - assert request2.meta["proxy"] == "https://b.example" - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_unproxied_http_to_unproxied_https(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("http://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "https://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "http://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_proxied_https_to_proxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request1.meta["_auth_proxy"] == "https://b.example" - assert request1.meta["proxy"] == "https://b.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request3.meta["_auth_proxy"] == "https://b.example" - assert request3.meta["proxy"] == "https://b.example" - - def test_system_proxy_proxied_https_to_unproxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "https_proxy": "https://b:@b.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request1.meta["_auth_proxy"] == "https://b.example" - assert request1.meta["proxy"] == "https://b.example" - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" - assert request3.meta["_auth_proxy"] == "https://b.example" - assert request3.meta["proxy"] == "https://b.example" - - def test_system_proxy_unproxied_https_to_proxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - env = { - "http_proxy": "https://a:@a.example", - } - with set_environ(**env): - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" - assert request2.meta["_auth_proxy"] == "https://a.example" - assert request2.meta["proxy"] == "https://a.example" - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - def test_system_proxy_unproxied_https_to_unproxied_http(self): - crawler = get_crawler() - redirect_mw = self.mwcls.from_crawler(crawler) - proxy_mw = HttpProxyMiddleware.from_crawler(crawler) - - request1 = Request("https://example.com") - proxy_mw.process_request(request1) - - assert "Proxy-Authorization" not in request1.headers - assert "_auth_proxy" not in request1.meta - assert "proxy" not in request1.meta - - response1 = self.get_response(request1, "http://example.com") - request2 = redirect_mw.process_response(request1, response1) - - assert isinstance(request2, Request) - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - proxy_mw.process_request(request2) - - assert "Proxy-Authorization" not in request2.headers - assert "_auth_proxy" not in request2.meta - assert "proxy" not in request2.meta - - response2 = self.get_response(request2, "https://example.com") - request3 = redirect_mw.process_response(request2, response2) - - assert isinstance(request3, Request) - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta - - proxy_mw.process_request(request3) - - assert "Proxy-Authorization" not in request3.headers - assert "_auth_proxy" not in request3.meta - assert "proxy" not in request3.meta +from tests.test_downloadermiddleware_redirect_base import ( + REDIRECT_SCHEME_CASES, + SCHEME_PARAMS, + Base, +) class TestRedirectMiddleware(Base.Test): @@ -1004,7 +36,7 @@ class TestRedirectMiddleware(Base.Test): return Response(request.url, status=status, headers=headers) def test_redirect_3xx_permanent(self): - def _test(method, status=301): + def _test(method, status: int): url = f"http://www.example.com/{status}" url2 = "http://www.example.com/redirected" req = Request(url, method=method) @@ -1019,10 +51,6 @@ class TestRedirectMiddleware(Base.Test): del rsp.headers["Location"] assert self.mw.process_response(req, rsp) is rsp - _test("GET") - _test("POST") - _test("HEAD") - _test("GET", status=307) _test("POST", status=307) _test("HEAD", status=307) @@ -1031,6 +59,212 @@ class TestRedirectMiddleware(Base.Test): _test("POST", status=308) _test("HEAD", status=308) + @pytest.mark.parametrize("status", [301, 302, 303]) + def test_method_becomes_get(self, status): + source_url = f"http://www.example.com/{status}" + target_url = "http://www.example.com/redirected2" + request = Request( + source_url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + response = Response(source_url, headers={"Location": target_url}, status=status) + redirect_request = self.mw.process_response(request, response) + assert isinstance(redirect_request, Request) + assert redirect_request.url == target_url + assert redirect_request.method == "GET" + assert "Content-Type" not in redirect_request.headers + assert "Content-Length" not in redirect_request.headers + assert not redirect_request.body + + @pytest.mark.parametrize("status", [301, 302]) + @pytest.mark.parametrize("method", ["PUT", "DELETE"]) + def test_method_not_converted_on_301_302(self, status, method): + url = f"http://www.example.com/{status}" + url2 = "http://www.example.com/redirected" + body = b"test-body" + req = Request( + url, + method=method, + body=body, + headers={"Content-Type": "text/plain", "Content-Length": str(len(body))}, + ) + rsp = Response(url, headers={"Location": url2}, status=status) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == method + assert req2.body == body + assert req2.headers[b"Content-Type"] == b"text/plain" + assert req2.headers[b"Content-Length"] == str(len(body)).encode() + + @pytest.mark.parametrize("method", ["PUT", "DELETE"]) + def test_method_converted_on_303(self, method): + status = 303 + url = f"http://www.example.com/{status}" + url2 = "http://www.example.com/redirected" + req = Request(url, method=method) + rsp = Response(url, headers={"Location": url2}, status=status) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + + def test_get_method_body_preserved_on_303(self): + status = 303 + url = f"http://www.example.com/{status}" + url2 = "http://www.example.com/redirected" + body = b"test-body" + req = Request( + url, + method="GET", + body=body, + headers={"Content-Type": "text/plain", "Content-Length": str(len(body))}, + ) + rsp = Response(url, headers={"Location": url2}, status=status) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + assert req2.body == body + assert req2.headers[b"Content-Type"] == b"text/plain" + assert req2.headers[b"Content-Length"] == str(len(body)).encode() + + def test_redirect_strips_content_headers(self): + url = "http://www.example.com/303" + url2 = "http://www.example.com/redirected" + headers = { + "Content-Type": "application/json", + "Content-Length": "100", + "Content-Encoding": "gzip", + "Content-Language": "en", + "Content-Location": "http://www.example.com/original", + "X-Custom": "foo", + } + req = Request(url, method="POST", headers=headers, body=b"foo") + rsp = Response(url, headers={"Location": url2}, status=303) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + assert req2.body == b"" + assert "Content-Type" not in req2.headers + assert "Content-Length" not in req2.headers + assert "Content-Encoding" not in req2.headers + assert "Content-Language" not in req2.headers + assert "Content-Location" not in req2.headers + assert req2.headers["X-Custom"] == b"foo" + + @pytest.mark.parametrize( + ( + "referer", + "source_url", + "target_url", + "policy_header", + "expected_referer", + ), + [ + ( + None, + "http://www.example.com/302", + "http://www.example.com/redirected", + None, + b"http://www.example.com/302", + ), + ( + "http://example.com/old", + "http://www.example.com/302", + "http://www.example.com/redirected", + None, + b"http://www.example.com/302", + ), + ( + "https://example.com/old", + "https://www.example.com/302", + "http://www.example.com/redirected", + None, + None, + ), + ( + "http://example.com/old", + "http://www.example.com/foo/bar", + "http://www.example.com/redirected", + "origin", + b"http://www.example.com/", + ), + ], + ) + def test_redirect_referer( + self, referer, source_url, target_url, policy_header, expected_referer + ): + headers = {"Referer": referer} if referer else {} + source_request = Request(source_url, headers=headers) + resp_headers = {"Location": target_url} + if policy_header: + resp_headers["Referrer-Policy"] = policy_header + response = Response(source_url, headers=resp_headers, status=302) + crawler = get_crawler() + referer_mw = build_from_crawler(RefererMiddleware, crawler) + redirect_mw = self.mwcls.from_crawler(crawler) + redirect_mw._referer_spider_middleware = referer_mw + redirect_request = redirect_mw.process_response(source_request, response) + if expected_referer: + assert redirect_request.headers.get("Referer") == expected_referer + else: + assert "Referer" not in redirect_request.headers + + def test_redirect_strips_referer_no_middleware(self): + source_url = "http://www.example.com/302" + redirect_url = "http://www.example.com/redirected" + source_request = Request( + source_url, headers={"Referer": "http://example.com/old"} + ) + response = Response(source_url, headers={"Location": redirect_url}, status=302) + redirect_mw = self.mwcls.from_crawler(get_crawler()) + redirect_mw._referer_spider_middleware = None + redirect_request = redirect_mw.process_response(source_request, response) + assert "Referer" not in redirect_request.headers + + @pytest.mark.parametrize("status", [307, 308]) + def test_cross_origin_maintain_body(self, status): + source_url = "https://example.com" + target_url = "https://attacker.example" + body = b"secret" + request = Request( + source_url, + method="POST", + body=body, + headers={ + "Content-Type": "application/json", + "Content-Length": str(len(body)), + }, + ) + response1 = Response( + source_url, headers={"Location": target_url}, status=status + ) + redirect_request = self.mw.process_response(request, response1) + assert isinstance(redirect_request, Request) + assert redirect_request.url == target_url + assert redirect_request.method == "POST" + assert redirect_request.body == body + assert redirect_request.headers[b"Content-Type"] == b"application/json" + assert redirect_request.headers[b"Content-Length"] == str(len(body)).encode() + + def test_redirect_keeps_fragment(self): + url = "http://www.example.com/301#frag" + url2 = "http://www.example.com/redirected" + req = Request(url) + rsp = Response(url, headers={"Location": url2}, status=302) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == "http://www.example.com/redirected#frag" + def test_redirect_302_head(self): url = "http://www.example.com/302" url2 = "http://www.example.com/redirected2" @@ -1107,44 +341,6 @@ class TestRedirectMiddleware(Base.Test): assert self.mw.process_response(request, response) is response -SCHEME_PARAMS = ("url", "location", "target") -HTTP_SCHEMES = ("http", "https") -NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo") -REDIRECT_SCHEME_CASES = ( - # http/https → http/https redirects - *( - ( - f"{input_scheme}://example.com/a", - f"{output_scheme}://example.com/b", - f"{output_scheme}://example.com/b", - ) - for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2) - ), - # http/https → data/file/ftp/s3/foo does not redirect - *( - ( - f"{input_scheme}://example.com/a", - f"{output_scheme}://example.com/b", - None, - ) - for input_scheme in HTTP_SCHEMES - for output_scheme in NON_HTTP_SCHEMES - ), - # http/https → relative redirects - *( - ( - f"{scheme}://example.com/a", - location, - f"{scheme}://example.com/b", - ) - for scheme in HTTP_SCHEMES - for location in ("//example.com/b", "/b") - ), - # Note: We do not test data/file/ftp/s3 schemes for the initial URL - # because their download handlers cannot return a status code of 3xx. -) - - @pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES) def test_redirect_schemes(url, location, target): crawler = get_crawler(Spider) @@ -1159,121 +355,106 @@ def test_redirect_schemes(url, location, target): assert redirect.url == target -def meta_refresh_body(url, interval=5): - html = f"""""" - return html.encode("utf-8") - - -class TestMetaRefreshMiddleware(Base.Test): - mwcls = MetaRefreshMiddleware - reason = "meta refresh" - - def setup_method(self): - crawler = get_crawler(Spider) - self.mw = self.mwcls.from_crawler(crawler) - - def _body(self, interval=5, url="http://example.org/newpage"): - return meta_refresh_body(url, interval) - - def get_response(self, request, location): - return HtmlResponse(request.url, body=self._body(url=location)) - - def test_meta_refresh(self): - req = Request(url="http://example.org") - rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp) - assert isinstance(req2, Request) - assert req2.url == "http://example.org/newpage" - - def test_meta_refresh_with_high_interval(self): - # meta-refresh with high intervals don't trigger redirects - req = Request(url="http://example.org") - rsp = HtmlResponse( - url="http://example.org", body=self._body(interval=1000), encoding="utf-8" - ) - rsp2 = self.mw.process_response(req, rsp) - assert rsp is rsp2 - - def test_meta_refresh_trough_posted_request(self): - req = Request( - url="http://example.org", - method="POST", - body="test", - headers={"Content-Type": "text/plain", "Content-length": "4"}, - ) - rsp = HtmlResponse(req.url, body=self._body()) - req2 = self.mw.process_response(req, rsp) - - assert isinstance(req2, Request) - assert req2.url == "http://example.org/newpage" - assert req2.method == "GET" - assert "Content-Type" not in req2.headers, ( - "Content-Type header must not be present in redirected request" - ) - assert "Content-Length" not in req2.headers, ( - "Content-Length header must not be present in redirected request" - ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" - - def test_ignore_tags_default(self): - req = Request(url="http://example.org") - body = ( - """""" - ) - rsp = HtmlResponse(req.url, body=body.encode()) - response = self.mw.process_response(req, rsp) - assert isinstance(response, Response) - - def test_ignore_tags_1_x_list(self): - """Test that Scrapy 1.x behavior remains possible""" - settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]} - crawler = get_crawler(Spider, settings) - mw = MetaRefreshMiddleware.from_crawler(crawler) - req = Request(url="http://example.org") - body = ( - """""" - ) - rsp = HtmlResponse(req.url, body=body.encode()) - response = mw.process_response(req, rsp) - assert isinstance(response, Response) - - @pytest.mark.parametrize( - SCHEME_PARAMS, + ("policy", "source_url", "target_url", "expected_referrer"), [ - *REDIRECT_SCHEME_CASES, - # data/file/ftp/s3/foo → * does not redirect - *( - ( - f"{input_scheme}://example.com/a", - f"{output_scheme}://example.com/b", - None, - ) - for input_scheme in NON_HTTP_SCHEMES - for output_scheme in chain(HTTP_SCHEMES, NON_HTTP_SCHEMES) + # The policy header affects the outcome. + # (without it, the https → http switch would drop the referer) + ( + POLICY_UNSAFE_URL, + "https://a.example/1", + "http://a.example/2", + b"https://a.example/1", ), - # data/file/ftp/s3/foo → relative does not redirect - *( - ( - f"{scheme}://example.com/a", - location, - None, - ) - for scheme in NON_HTTP_SCHEMES - for location in ("//example.com/b", "/b") + # The policy header can get the Referer header removed. + ( + POLICY_NO_REFERRER, + "http://a.example/1", + "http://a.example/2", + None, + ), + # The policy header can get the Referer header edited (path stripped). + ( + POLICY_ORIGIN, + "http://a.example/1", + "http://a.example/2", + b"http://a.example/", ), ], ) -def test_meta_refresh_schemes(url, location, target): - crawler = get_crawler(Spider) - mw = MetaRefreshMiddleware.from_crawler(crawler) - request = Request(url) - response = HtmlResponse(url, body=meta_refresh_body(location)) - redirect = mw.process_response(request, response) - if target is None: - assert redirect == response - else: - assert isinstance(redirect, Request) - assert redirect.url == target +def test_response_referrer_policy(policy, source_url, target_url, expected_referrer): + crawler = get_crawler() + referrer_mw = build_from_crawler(RefererMiddleware, crawler) + redirect_mw = build_from_crawler(RedirectMiddleware, crawler) + redirect_mw._referer_spider_middleware = referrer_mw + source_request = Request(source_url) + extra_headers = {} + if policy: + extra_headers["Referrer-Policy"] = policy + response_redirect = Response( + source_request.url, + status=301, + headers={"Location": target_url, **extra_headers}, + ) + source_request = redirect_mw.process_response(source_request, response_redirect) + assert isinstance(source_request, Request) + + assert source_request.headers.get("Referer") == expected_referrer + + +def test_no_warning_when_referer_middleware_present(caplog): + crawler = get_crawler() + crawler.get_spider_middleware = MagicMock(return_value=MagicMock()) + mw = build_from_crawler(RedirectMiddleware, crawler) + caplog.clear() + with caplog.at_level(logging.WARNING): + mw._engine_started() + assert not [ + record + for record in caplog.records + if record.name == "scrapy.downloadermiddlewares.redirect" + ] + + +def test_warning_redirect_middleware(caplog): + crawler = get_crawler() + crawler.get_spider_middleware = MagicMock(return_value=None) + mw = build_from_crawler(RedirectMiddleware, crawler) + with caplog.at_level(logging.WARNING): + mw._engine_started() + assert ( + "scrapy.downloadermiddlewares.redirect.RedirectMiddleware found no " + "scrapy.spidermiddlewares.referer.RefererMiddleware" + ) in caplog.text + assert ( + "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" + in caplog.text + ) + assert ( + "replace scrapy.downloadermiddlewares.redirect.RedirectMiddleware " + "with a subclass that overrides the handle_referer() method" + ) in caplog.text + + +def test_warning_subclass(caplog): + class MyRedirectMiddleware(RedirectMiddleware): + pass + + crawler = get_crawler() + crawler.get_spider_middleware = MagicMock(return_value=None) + mw = build_from_crawler(MyRedirectMiddleware, crawler) + with caplog.at_level(logging.WARNING): + mw._engine_started() + assert ( + "test_warning_subclass..MyRedirectMiddleware found no " + "scrapy.spidermiddlewares.referer.RefererMiddleware" + ) in caplog.text + assert ( + "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" + in caplog.text + ) + assert "edit " in caplog.text + assert "test_warning_subclass..MyRedirectMiddleware" in caplog.text + assert ( + "(if defined in your code base) to override the handle_referer() method" + ) in caplog.text diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py new file mode 100644 index 000000000..32935769f --- /dev/null +++ b/tests/test_downloadermiddleware_redirect_base.py @@ -0,0 +1,1023 @@ +from __future__ import annotations + +from itertools import product + +import pytest + +from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware +from scrapy.exceptions import IgnoreRequest +from scrapy.http import Request, Response +from scrapy.utils.misc import set_environ +from scrapy.utils.test import get_crawler + +SCHEME_PARAMS = ("url", "location", "target") +HTTP_SCHEMES = ("http", "https") +NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo") +REDIRECT_SCHEME_CASES = ( + # http/https → http/https redirects + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + f"{output_scheme}://example.com/b", + ) + for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2) + ), + # http/https → data/file/ftp/s3/foo does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in HTTP_SCHEMES + for output_scheme in NON_HTTP_SCHEMES + ), + # http/https → relative redirects + *( + ( + f"{scheme}://example.com/a", + location, + f"{scheme}://example.com/b", + ) + for scheme in HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + # Note: We do not test data/file/ftp/s3 schemes for the initial URL + # because their download handlers cannot return a status code of 3xx. +) + + +class Base: + class Test: + def test_priority_adjust(self): + req = Request("http://a.example") + rsp = self.get_response(req, "http://a.example/redirected") + req2 = self.mw.process_response(req, rsp) + assert req2.priority > req.priority + + def test_dont_redirect(self): + url = "http://www.example.com/301" + url2 = "http://www.example.com/redirected" + req = Request(url, meta={"dont_redirect": True}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Response) + assert r is rsp + + # Test that it redirects when dont_redirect is False + req = Request(url, meta={"dont_redirect": False}) + rsp = self.get_response(req, url2) + + r = self.mw.process_response(req, rsp) + assert isinstance(r, Request) + + def test_post(self): + url = "http://www.example.com/302" + url2 = "http://www.example.com/redirected2" + req = Request( + url, + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = self.get_response(req, url2) + + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == url2 + assert req2.method == "GET" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + + def test_max_redirect_times(self): + self.mw.max_redirect_times = 1 + req = Request("http://a.example/302") + rsp = self.get_response(req, "/redirected") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + assert "redirect_times" in req.meta + assert req.meta["redirect_times"] == 1 + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_ttl(self): + self.mw.max_redirect_times = 100 + req = Request("http://a.example/302", meta={"redirect_ttl": 1}) + rsp = self.get_response(req, "/a") + + req = self.mw.process_response(req, rsp) + assert isinstance(req, Request) + with pytest.raises(IgnoreRequest): + self.mw.process_response(req, rsp) + + def test_redirect_urls(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + + assert req2.url == "http://a.example/redirected" + assert req2.meta["redirect_urls"] == ["http://a.example/first"] + assert req3.url == "http://a.example/redirected2" + assert req3.meta["redirect_urls"] == [ + "http://a.example/first", + "http://a.example/redirected", + ] + + def test_redirect_reasons(self): + req1 = Request("http://a.example/first") + rsp1 = self.get_response(req1, "/redirected1") + req2 = self.mw.process_response(req1, rsp1) + rsp2 = self.get_response(req2, "/redirected2") + req3 = self.mw.process_response(req2, rsp2) + assert req2.meta["redirect_reasons"] == [self.reason] + assert req3.meta["redirect_reasons"] == [self.reason, self.reason] + + def test_cross_origin_header_dropping(self): + safe_headers = {"A": "B"} + cookie_header = {"Cookie": "a=b"} + authorization_header = {"Authorization": "Bearer 123456"} + + original_request = Request( + "https://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers. + internal_response = self.get_response( + original_request, "https://example.com/a" + ) + internal_redirect_request = self.mw.process_response( + original_request, internal_response + ) + assert isinstance(internal_redirect_request, Request) + assert original_request.headers == internal_redirect_request.headers + + # Redirects to the same origin (same scheme, same domain, same port) + # keep all headers also when the scheme is http. + http_request = Request( + "http://example.com", + headers={**safe_headers, **cookie_header, **authorization_header}, + ) + http_response = self.get_response(http_request, "http://example.com/a") + http_redirect_request = self.mw.process_response( + http_request, http_response + ) + assert isinstance(http_redirect_request, Request) + assert http_request.headers == http_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_explicit_port_response = self.get_response( + original_request, "https://example.com:443/a" + ) + to_explicit_port_redirect_request = self.mw.process_response( + original_request, to_explicit_port_response + ) + assert isinstance(to_explicit_port_redirect_request, Request) + assert original_request.headers == to_explicit_port_redirect_request.headers + + # For default ports, whether the port is explicit or implicit does not + # affect the outcome, it is still the same origin. + to_implicit_port_response = self.get_response( + original_request, "https://example.com/a" + ) + to_implicit_port_redirect_request = self.mw.process_response( + original_request, to_implicit_port_response + ) + assert isinstance(to_implicit_port_redirect_request, Request) + assert original_request.headers == to_implicit_port_redirect_request.headers + + # A port change drops the Authorization header because the origin + # changes, but keeps the Cookie header because the domain remains the + # same. + different_port_response = self.get_response( + original_request, "https://example.com:8080/a" + ) + different_port_redirect_request = self.mw.process_response( + original_request, different_port_response + ) + assert isinstance(different_port_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == different_port_redirect_request.headers.to_unicode_dict() + + # A domain change drops both the Authorization and the Cookie header. + external_response = self.get_response( + original_request, "https://example.org/a" + ) + external_redirect_request = self.mw.process_response( + original_request, external_response + ) + assert isinstance(external_redirect_request, Request) + assert safe_headers == external_redirect_request.headers.to_unicode_dict() + + # A scheme upgrade (http → https) drops the Authorization header + # because the origin changes, but keeps the Cookie header because the + # domain remains the same. + upgrade_response = self.get_response(http_request, "https://example.com/a") + upgrade_redirect_request = self.mw.process_response( + http_request, upgrade_response + ) + assert isinstance(upgrade_redirect_request, Request) + assert { + **safe_headers, + **cookie_header, + } == upgrade_redirect_request.headers.to_unicode_dict() + + # A scheme downgrade (https → http) drops the Authorization header + # because the origin changes, and the Cookie header because its value + # cannot indicate whether the cookies were secure (HTTPS-only) or not. + # + # Note: If the Cookie header is set by the cookie management + # middleware, as recommended in the docs, the dropping of Cookie on + # scheme downgrade is not an issue, because the cookie management + # middleware will add again the Cookie header to the new request if + # appropriate. + downgrade_response = self.get_response( + original_request, "http://example.com/a" + ) + downgrade_redirect_request = self.mw.process_response( + original_request, downgrade_response + ) + assert isinstance(downgrade_redirect_request, Request) + assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict() + + def test_meta_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_http_to_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("http://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_meta_proxy_https_to_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + meta = {"proxy": "https://a:@a.example"} + request1 = Request("https://example.com", meta=meta) + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_http_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_absolute(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_https_relative(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "/a") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "/a") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_proxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request1.meta["_auth_proxy"] == "https://a.example" + assert request1.meta["proxy"] == "https://a.example" + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request3.meta["_auth_proxy"] == "https://a.example" + assert request3.meta["proxy"] == "https://a.example" + + def test_system_proxy_unproxied_http_to_proxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request2.meta["_auth_proxy"] == "https://b.example" + assert request2.meta["proxy"] == "https://b.example" + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_http_to_unproxied_https(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("http://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "https://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "http://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_proxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_proxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "https_proxy": "https://b:@b.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert request1.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request1.meta["_auth_proxy"] == "https://b.example" + assert request1.meta["proxy"] == "https://b.example" + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert request3.headers["Proxy-Authorization"] == b"Basic Yjo=" + assert request3.meta["_auth_proxy"] == "https://b.example" + assert request3.meta["proxy"] == "https://b.example" + + def test_system_proxy_unproxied_https_to_proxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + env = { + "http_proxy": "https://a:@a.example", + } + with set_environ(**env): + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert request2.headers["Proxy-Authorization"] == b"Basic YTo=" + assert request2.meta["_auth_proxy"] == "https://a.example" + assert request2.meta["proxy"] == "https://a.example" + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + def test_system_proxy_unproxied_https_to_unproxied_http(self): + crawler = get_crawler() + redirect_mw = self.mwcls.from_crawler(crawler) + proxy_mw = HttpProxyMiddleware.from_crawler(crawler) + + request1 = Request("https://example.com") + proxy_mw.process_request(request1) + + assert "Proxy-Authorization" not in request1.headers + assert "_auth_proxy" not in request1.meta + assert "proxy" not in request1.meta + + response1 = self.get_response(request1, "http://example.com") + request2 = redirect_mw.process_response(request1, response1) + + assert isinstance(request2, Request) + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + proxy_mw.process_request(request2) + + assert "Proxy-Authorization" not in request2.headers + assert "_auth_proxy" not in request2.meta + assert "proxy" not in request2.meta + + response2 = self.get_response(request2, "https://example.com") + request3 = redirect_mw.process_response(request2, response2) + + assert isinstance(request3, Request) + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta + + proxy_mw.process_request(request3) + + assert "Proxy-Authorization" not in request3.headers + assert "_auth_proxy" not in request3.meta + assert "proxy" not in request3.meta diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py new file mode 100644 index 000000000..416fbc2aa --- /dev/null +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import logging +from itertools import chain +from unittest.mock import MagicMock + +import pytest + +from scrapy.downloadermiddlewares.redirect import MetaRefreshMiddleware +from scrapy.http import HtmlResponse, Request, Response +from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler +from tests.test_downloadermiddleware_redirect_base import ( + HTTP_SCHEMES, + NON_HTTP_SCHEMES, + REDIRECT_SCHEME_CASES, + SCHEME_PARAMS, + Base, +) + + +def meta_refresh_body(url, interval=5): + html = f"""""" + return html.encode("utf-8") + + +class TestMetaRefreshMiddleware(Base.Test): + mwcls = MetaRefreshMiddleware + reason = "meta refresh" + + def setup_method(self): + crawler = get_crawler(Spider) + self.mw = self.mwcls.from_crawler(crawler) + + def _body(self, interval=5, url="http://example.org/newpage"): + return meta_refresh_body(url, interval) + + def get_response(self, request, location): + return HtmlResponse(request.url, body=self._body(url=location)) + + def test_meta_refresh(self): + req = Request(url="http://example.org") + rsp = HtmlResponse(req.url, body=self._body()) + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.url == "http://example.org/newpage" + + def test_meta_refresh_with_high_interval(self): + # meta-refresh with high intervals don't trigger redirects + req = Request(url="http://example.org") + rsp = HtmlResponse( + url="http://example.org", body=self._body(interval=1000), encoding="utf-8" + ) + rsp2 = self.mw.process_response(req, rsp) + assert rsp is rsp2 + + def test_meta_refresh_trough_posted_request(self): + req = Request( + url="http://example.org", + method="POST", + body="test", + headers={"Content-Type": "text/plain", "Content-length": "4"}, + ) + rsp = HtmlResponse(req.url, body=self._body()) + req2 = self.mw.process_response(req, rsp) + + assert isinstance(req2, Request) + assert req2.url == "http://example.org/newpage" + assert req2.method == "GET" + assert "Content-Type" not in req2.headers, ( + "Content-Type header must not be present in redirected request" + ) + assert "Content-Length" not in req2.headers, ( + "Content-Length header must not be present in redirected request" + ) + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + + def test_ignore_tags_default(self): + req = Request(url="http://example.org") + body = ( + """""" + ) + rsp = HtmlResponse(req.url, body=body.encode()) + response = self.mw.process_response(req, rsp) + assert isinstance(response, Response) + + def test_ignore_tags_1_x_list(self): + """Test that Scrapy 1.x behavior remains possible""" + settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]} + crawler = get_crawler(Spider, settings) + mw = MetaRefreshMiddleware.from_crawler(crawler) + req = Request(url="http://example.org") + body = ( + """""" + ) + rsp = HtmlResponse(req.url, body=body.encode()) + response = mw.process_response(req, rsp) + assert isinstance(response, Response) + + +@pytest.mark.parametrize( + SCHEME_PARAMS, + [ + *REDIRECT_SCHEME_CASES, + # data/file/ftp/s3/foo → * does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in NON_HTTP_SCHEMES + for output_scheme in chain(HTTP_SCHEMES, NON_HTTP_SCHEMES) + ), + # data/file/ftp/s3/foo → relative does not redirect + *( + ( + f"{scheme}://example.com/a", + location, + None, + ) + for scheme in NON_HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + ], +) +def test_meta_refresh_schemes(url, location, target): + crawler = get_crawler(Spider) + mw = MetaRefreshMiddleware.from_crawler(crawler) + request = Request(url) + response = HtmlResponse(url, body=meta_refresh_body(location)) + redirect = mw.process_response(request, response) + if target is None: + assert redirect == response + else: + assert isinstance(redirect, Request) + + +def test_warning_meta_refresh_middleware(caplog): + crawler = get_crawler() + crawler.get_spider_middleware = MagicMock(return_value=None) + mw = build_from_crawler(MetaRefreshMiddleware, crawler) + with caplog.at_level(logging.WARNING): + mw._engine_started() + assert ( + "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware found no " + "scrapy.spidermiddlewares.referer.RefererMiddleware" + ) in caplog.text + assert ( + "enable scrapy.spidermiddlewares.referer.RefererMiddleware (or a subclass)" + in caplog.text + ) + assert ( + "replace scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware " + "with a subclass that overrides the handle_referer() method" + ) in caplog.text diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 50946899a..56d21a4d2 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -84,6 +84,39 @@ class TestRetry: ) assert self.crawler.stats.get_value("retry/count") == 2 + def test_give_up_log_level_setting(self): + crawler = get_crawler( + DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"} + ) + crawler.spider = crawler._create_spider() + mw = RetryMiddleware.from_crawler(crawler) + mw.max_retry_times = 0 + req = Request("http://example.com/503") + rsp = Response("http://example.com/503", body=b"", status=503) + with LogCapture() as log: + assert mw.process_response(req, rsp) is rsp + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", + ) + ) + + def test_give_up_log_level_meta(self): + self.mw.max_retry_times = 0 + req = Request("http://example.com/503", meta={"give_up_log_level": "WARNING"}) + rsp = Response("http://example.com/503", body=b"", status=503) + with LogCapture() as log: + assert self.mw.process_response(req, rsp) is rsp + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", + ) + ) + def test_twistederrors(self): exceptions = [ ConnectError, @@ -612,6 +645,87 @@ class TestGetRetryRequest: ) ) + def test_give_up_log_level_default(self): + request = Request("https://example.com") + spider = self.get_spider() + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_argument_name(self): + request = Request("https://example.com") + spider = self.get_spider() + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + give_up_log_level="WARNING", + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_argument_number(self): + request = Request("https://example.com") + spider = self.get_spider() + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + give_up_log_level=logging.WARNING, + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_setting(self): + request = Request("https://example.com") + spider = self.get_spider({"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}) + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_invalid(self): + request = Request("https://example.com") + spider = self.get_spider() + with pytest.raises(ValueError, match="Invalid give-up log level"): + get_retry_request( + request, + spider=spider, + max_retry_times=0, + give_up_log_level="NOT_A_LEVEL", + ) + def test_custom_stats_key(self): request = Request("https://example.com") spider = self.get_spider() diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 15c68779d..082fc743e 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -23,8 +23,8 @@ if TYPE_CHECKING: class TestRobotsTxtMiddleware: - def setup_method(self): - self.crawler = mock.MagicMock() + def setup_method(self) -> None: + self.crawler: mock.MagicMock = mock.MagicMock() self.crawler.settings = Settings() self.crawler.engine.download_async = mock.AsyncMock() diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 67af4264c..cf7b614c4 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -16,7 +16,7 @@ class TestDownloaderStats: self.crawler.stats.open_spider() self.req = Request("http://scrapytest.org") - self.res = Response("scrapytest.org", status=400) + self.res = Response("http://scrapytest.org", status=400) def assertStatsEqual(self, key, value): assert self.crawler.stats.get_value(key) == value, str( diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 5c31f68bc..9d58a6e09 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -5,7 +5,6 @@ import pytest from scrapy import Request from scrapy.core.downloader import Downloader, Slot -from scrapy.crawler import CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler @@ -33,6 +32,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) + assert self.mockserver self.default_slot = self.mockserver.host self.times: dict[str, list[float]] = {} @@ -64,9 +64,6 @@ class TestCrawl: def teardown_class(cls): cls.mockserver.__exit__(None, None, None) - def setup_method(self): - self.runner = CrawlerRunner() - @inline_callbacks_test def test_delay(self): crawler = get_crawler(DownloaderSlotsSettingsTestSpider) @@ -84,8 +81,8 @@ class TestCrawl: assert max(list(error_delta.values())) < tolerance -@pytest.mark.requires_reactor # needs a reactor or an event loop for Downloader._slot_gc_loop -def test_params(): +@coroutine_test +async def test_params(): params = { "concurrency": 1, "delay": 2, @@ -109,8 +106,8 @@ def test_params(): ) -@pytest.mark.requires_reactor # needs a reactor or an event loop for Downloader._slot_gc_loop -def test_get_slot_deprecated_spider_arg(): +@coroutine_test +async def test_get_slot_deprecated_spider_arg(): crawler = get_crawler(DefaultSpider) crawler.spider = crawler._create_spider() downloader = Downloader(crawler) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index b38bf9570..5d79691b2 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -3,8 +3,8 @@ import shutil import sys import tempfile from pathlib import Path -from warnings import catch_warnings +import pytest from testfixtures import LogCapture from scrapy.core.scheduler import Scheduler @@ -86,7 +86,7 @@ class TestRFPDupeFilter: df.close("finished") df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False) - assert df != df2 + assert df is not df2 try: df2.open() assert df2.request_seen(r1) @@ -260,11 +260,8 @@ class TestBaseDupeFilter: dupefilter = _get_dupefilter( settings={"DUPEFILTER_CLASS": BaseDupeFilter}, ) - with catch_warnings(record=True) as warning_list: + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Calling BaseDupeFilter\.log\(\) is deprecated.", + ): dupefilter.log(None, None) - assert len(warning_list) == 1 - assert ( - str(warning_list[0].message) - == "Calling BaseDupeFilter.log() is deprecated." - ) - assert warning_list[0].category == ScrapyDeprecationWarning diff --git a/tests/test_engine.py b/tests/test_engine.py index 4573c5001..e51eb4664 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -6,8 +6,7 @@ import subprocess import sys from collections import defaultdict from dataclasses import dataclass -from logging import DEBUG -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, cast from unittest.mock import Mock, call from urllib.parse import urlparse @@ -22,10 +21,11 @@ from scrapy import signals from scrapy.core.engine import ExecutionEngine, _Slot from scrapy.core.scheduler import BaseScheduler from scrapy.exceptions import CloseSpider, IgnoreRequest -from scrapy.http import Request, Response +from scrapy.http import Headers, Request, Response from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider +from scrapy.statscollectors import MemoryStatsCollector from scrapy.utils.defer import ( _schedule_coro, deferred_from_coro, @@ -38,9 +38,10 @@ from tests import get_testdata from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: + from twisted.python.failure import Failure + from scrapy.core.scheduler import Scheduler from scrapy.crawler import Crawler - from scrapy.statscollectors import MemoryStatsCollector from tests.mockserver.http import MockServer @@ -133,16 +134,16 @@ class ChangeCloseReasonSpider(MySpider): class CrawlerRun: """A class to run the crawler and keep track of events occurred""" - def __init__(self, spider_class): - self.respplug = [] - self.reqplug = [] - self.reqdropped = [] - self.reqreached = [] - self.itemerror = [] - self.itemresp = [] - self.headers = {} - self.bytes = defaultdict(list) - self.signals_caught = {} + def __init__(self, spider_class: type[Spider]): + self.respplug: list[tuple[Response, Spider]] = [] + self.reqplug: list[tuple[Request, Spider]] = [] + self.reqdropped: list[tuple[Request, Spider]] = [] + self.reqreached: list[tuple[Request, Spider]] = [] + self.itemerror: list[tuple[Any, Response, Spider, Failure]] = [] + self.itemresp: list[tuple[Any, Response]] = [] + self.headers: dict[Request, Headers] = {} + self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list) + self.signals_caught: dict[Any, dict[str, Any]] = {} self.spider_class = spider_class async def run(self, mockserver: MockServer) -> None: @@ -188,35 +189,39 @@ class CrawlerRun: def geturl(self, path: str) -> str: return self.mockserver.url(path) - def getpath(self, url): + def getpath(self, url: str) -> str: u = urlparse(url) return u.path - def item_error(self, item, response, spider, failure): + def item_error( + self, item: Any, response: Response, spider: Spider, failure: Failure + ) -> None: self.itemerror.append((item, response, spider, failure)) - def item_scraped(self, item, spider, response): + def item_scraped(self, item: Any, spider: Spider, response: Response) -> None: self.itemresp.append((item, response)) - def headers_received(self, headers, body_length, request, spider): + def headers_received( + self, headers: Headers, body_length: int, request: Request, spider: Spider + ) -> None: self.headers[request] = headers - def bytes_received(self, data, request, spider): + def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None: self.bytes[request].append(data) - def request_scheduled(self, request, spider): + def request_scheduled(self, request: Request, spider: Spider) -> None: self.reqplug.append((request, spider)) - def request_reached(self, request, spider): + def request_reached(self, request: Request, spider: Spider) -> None: self.reqreached.append((request, spider)) - def request_dropped(self, request, spider): + def request_dropped(self, request: Request, spider: Spider) -> None: self.reqdropped.append((request, spider)) - def response_downloaded(self, response, spider): + def response_downloaded(self, response: Response, spider: Spider) -> None: self.respplug.append((response, spider)) - def record_signal(self, *args, **kwargs): + def record_signal(self, *args: Any, **kwargs: Any) -> None: """Record a signal and its parameters""" signalargs = kwargs.copy() sig = signalargs.pop("signal") @@ -294,8 +299,8 @@ class TestEngineBase: @staticmethod def _assert_scraped_items(run: CrawlerRun) -> None: assert len(run.itemresp) == 2 - for item, response in run.itemresp: - item = ItemAdapter(item) + for item_, response in run.itemresp: + item = ItemAdapter(item_) assert item["url"] == response.url if "item1.html" in item["url"]: assert item["name"] == "Item 1 name" @@ -308,6 +313,7 @@ class TestEngineBase: def _assert_headers_received(run: CrawlerRun) -> None: for headers in run.headers.values(): assert b"Server" in headers + assert headers[b"Server"] assert b"TwistedWeb" in headers[b"Server"] assert b"Date" in headers assert b"Content-Type" in headers @@ -388,6 +394,7 @@ class TestEngine(TestEngineBase): self._assert_downloaded_responses(run, count=9) self._assert_scraped_items(run) self._assert_signals_caught(run) + self._assert_headers_received(run) self._assert_bytes_received(run) @coroutine_test @@ -598,8 +605,8 @@ class TestEngineDownload(TestEngineDownloadAsync): return await maybe_deferred_to_future(engine.download(request)) -@pytest.mark.requires_reactor # needs a reactor or an event loop for _Slot.heartbeat -def test_request_scheduled_signal(caplog): +@coroutine_test +async def test_request_scheduled_signal(): class TestScheduler(BaseScheduler): def __init__(self): self.enqueued = [] @@ -626,7 +633,6 @@ def test_request_scheduled_signal(caplog): keep_request = Request("https://keep.example") engine._schedule_request(keep_request) drop_request = Request("https://drop.example") - caplog.set_level(DEBUG) engine._schedule_request(drop_request) assert scheduler.enqueued == [keep_request], ( f"{scheduler.enqueued!r} != [{keep_request!r}]" @@ -734,7 +740,8 @@ class TestEngineCloseSpider: engine = ExecutionEngine(crawler, lambda _: None) crawler.engine = engine await engine.open_spider_async() - del cast("MemoryStatsCollector", crawler.stats).spider_stats + assert isinstance(crawler.stats, MemoryStatsCollector) + del crawler.stats.spider_stats await engine.close_spider_async() assert "Stats close failure" in caplog.text diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 115deb821..6cc8c0650 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -4,30 +4,21 @@ from collections import deque from logging import ERROR from typing import TYPE_CHECKING -import pytest -from twisted.internet.defer import Deferred - from scrapy import Request, Spider, signals -from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.asyncio import call_later from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.test_scheduler import MemoryScheduler +from tests.utils import async_sleep from tests.utils.decorators import coroutine_test if TYPE_CHECKING: + import pytest + from scrapy.http import Response -async def sleep(seconds: float = 0.001) -> None: - from twisted.internet import reactor - - deferred: Deferred[None] = Deferred() - reactor.callLater(seconds, deferred.callback, None) - await maybe_deferred_to_future(deferred) - - class TestMain: - @pytest.mark.requires_reactor # TODO @coroutine_test async def test_sleep(self): """Neither asynchronous sleeps on Spider.start() nor the equivalent on @@ -40,27 +31,25 @@ class TestMain: name = "test" async def start(self): - from twisted.internet import reactor - yield Request("data:,a") - await sleep(seconds) + await async_sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) # During this time, the scheduler reports having requests but # returns None. - await sleep(seconds) + await async_sleep(seconds) self.crawler.engine._slot.scheduler.unpause() # The scheduler request is processed. - await sleep(seconds) + await async_sleep(seconds) yield Request("data:,c") - await sleep(seconds) + await async_sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) @@ -69,7 +58,7 @@ class TestMain: # delayed call below, proving that the start iteration can # finish before a scheduler “sleep” without causing the # scheduler to finish. - reactor.callLater(seconds, self.crawler.engine._slot.scheduler.unpause) + call_later(seconds, self.crawler.engine._slot.scheduler.unpause) def parse(self, response): pass @@ -274,7 +263,7 @@ class TestRequestSendOrder: @coroutine_test async def test_shared_queues(self): """If SCHEDULER_START_*_QUEUE is falsy, start requests and other - requests share the same queue, i.e. start requests are not priorized + requests share the same queue, i.e. start requests are not prioritized over other requests if their priority matches.""" nums = list(range(1, 14)) response_seconds = 0 diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py new file mode 100644 index 000000000..a474725d8 --- /dev/null +++ b/tests/test_extension_memusage.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import logging +import sys + +import pytest + +from scrapy import signals +from scrapy.core import engine as engine_mod +from scrapy.exceptions import NotConfigured +from scrapy.extensions import memusage as memusage_mod +from scrapy.extensions.memusage import MemoryUsage +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from tests.utils import OneShotLoop +from tests.utils.decorators import coroutine_test + +# MemoryUsage relies on the stdlib 'resource' module (not available on Windows) +pytestmark = pytest.mark.skipif( + sys.platform.startswith("win"), + reason="MemoryUsage extension not available on Windows", +) + + +MB = 1024 * 1024 + + +class _LoopSpider(Spider): + name = "loop-data-spider" + + def __init__(self, url: str, loops: int = 60, **kw): + super().__init__(**kw) + self.url = url + self.loops = loops + self.start_urls = [url] + + def parse(self, response): + count = response.meta.get("count", 0) + if count + 1 < self.loops: + yield response.follow( + self.url, callback=self.parse, meta={"count": count + 1} + ) + + +def test_memusage_disabled() -> None: + settings = { + "MEMUSAGE_ENABLED": False, + } + with pytest.raises(NotConfigured): + MemoryUsage.from_crawler(get_crawler(settings_dict=settings)) + + +@coroutine_test +async def test_memusage_limit_closes_spider_with_reason_and_error_log( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = { + "MEMUSAGE_LIMIT_MB": 10, + "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01, + "TELNETCONSOLE_ENABLED": False, + "LOG_LEVEL": "INFO", + } + + # Avoid background LoopingCall that can log after the test finishes. + monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + # Avoid engine start/stop races (the extension stops the engine in engine_started). + monkeypatch.setattr(engine_mod, "create_looping_call", OneShotLoop) + monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 250 * MB) + + crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) + + with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"): + await crawler.crawl_async(url="data:,", loops=100) + + assert crawler.stats + assert crawler.stats.get_value("memusage/limit_reached") == 1 + assert crawler.stats.get_value("finish_reason") == "memusage_exceeded" + assert any( + "memory usage exceeded" in r.getMessage().lower() for r in caplog.records + ) + + +@coroutine_test +async def test_memusage_warning_logs_but_allows_normal_finish( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = { + "MEMUSAGE_WARNING_MB": 50, + "MEMUSAGE_LIMIT_MB": 0, # no hard limit + "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01, + "TELNETCONSOLE_ENABLED": False, + "LOG_LEVEL": "INFO", + } + + # Avoid background LoopingCall that can log after the test finishes. + monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop) + monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB) + + crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings) + + warning_signals: list[int] = [] + + def on_warning_reached() -> None: + warning_signals.append(1) + + crawler.signals.connect(on_warning_reached, signal=signals.memusage_warning_reached) + + with caplog.at_level(logging.WARNING, logger="scrapy.extensions.memusage"): + await crawler.crawl_async(url="data:,", loops=60) + + assert warning_signals == [1] + assert crawler.stats + assert crawler.stats.get_value("memusage/warning_reached") == 1 + assert crawler.stats.get_value("finish_reason") == "finished" + assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records) diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 0e2d4be28..18782fb70 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -3,12 +3,11 @@ from __future__ import annotations import datetime from typing import TYPE_CHECKING, Any -import pytest - from scrapy.extensions.periodic_log import PeriodicLog from scrapy.utils.test import get_crawler from .spiders import MetaSpider +from .utils.decorators import coroutine_test if TYPE_CHECKING: from collections.abc import Callable @@ -59,10 +58,10 @@ stats_dump_2 = { class CustomPeriodicLog(PeriodicLog): - def set_a(self): + def set_a(self) -> None: self.stats._stats = stats_dump_1 - def set_b(self): + def set_b(self) -> None: self.stats._stats = stats_dump_2 @@ -88,15 +87,17 @@ class TestPeriodicLog: assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) - @pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.task - def test_log_delta(self): - def emulate(settings=None): + @coroutine_test + async def test_log_delta(self): + def emulate( + settings: dict[str, Any] | None = None, + ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]: spider = MetaSpider() ext = extension(settings) ext.spider_opened(spider) ext.set_a() a = ext.log_delta() - ext.set_a() + ext.set_b() b = ext.log_delta() ext.spider_closed(spider, reason="finished") return ext, a, b @@ -152,15 +153,17 @@ class TestPeriodicLog: ), ) - @pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.task - def test_log_stats(self): - def emulate(settings=None): + @coroutine_test + async def test_log_stats(self): + def emulate( + settings: dict[str, Any] | None = None, + ) -> tuple[PeriodicLog, dict[str, Any], dict[str, Any]]: spider = MetaSpider() ext = extension(settings) ext.spider_opened(spider) ext.set_a() a = ext.log_crawler_stats() - ext.set_a() + ext.set_b() b = ext.log_crawler_stats() ext.spider_closed(spider, reason="finished") return ext, a, b diff --git a/tests/test_extension_statsmailer.py b/tests/test_extension_statsmailer.py index d2e4d5c00..4d208a1ea 100644 --- a/tests/test_extension_statsmailer.py +++ b/tests/test_extension_statsmailer.py @@ -1,15 +1,28 @@ +import warnings from unittest.mock import MagicMock import pytest from scrapy import signals -from scrapy.exceptions import NotConfigured -from scrapy.extensions import statsmailer -from scrapy.mail import MailSender +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.signalmanager import SignalManager from scrapy.statscollectors import StatsCollector from scrapy.utils.spider import DefaultSpider +with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + r"The scrapy\.extensions\.statsmailer module is deprecated", + ScrapyDeprecationWarning, + ) + warnings.filterwarnings( + "ignore", + r"The scrapy\.mail module is deprecated", + ScrapyDeprecationWarning, + ) + from scrapy.extensions import statsmailer + from scrapy.mail import MailSender + @pytest.fixture def dummy_stats(): diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index a956ab184..3f9135867 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -6,7 +6,7 @@ from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler from tests.utils.decorators import inline_callbacks_test -pytestmark = pytest.mark.requires_reactor +pytestmark = pytest.mark.requires_reactor # TelnetConsole requires a reactor class TestTelnetExtension: @@ -53,3 +53,9 @@ class TestTelnetExtension: d = portal.login(creds, None, ITelnetProtocol) yield d console.stop_listening() + + def test_invalid_reversed_portrange(self): + settings = {"TELNETCONSOLE_PORT": [2, 1]} + console = TelnetConsole(get_crawler(settings_dict=settings)) + with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"): + console.start_listening() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1ca5f21f0..c1d6f04eb 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,608 +1,54 @@ from __future__ import annotations -import bz2 import csv -import gzip import json -import lzma import marshal -import os import pickle import random import shutil -import string -import sys import tempfile -import warnings from abc import ABC, abstractmethod -from collections import defaultdict -from io import BytesIO from logging import getLogger from pathlib import Path from string import ascii_letters, digits from typing import IO, TYPE_CHECKING, Any from unittest import mock -from urllib.parse import quote, urljoin +from urllib.parse import urljoin from urllib.request import pathname2url import lxml.etree import pytest -from packaging.version import Version from testfixtures import LogCapture -from w3lib.url import file_uri_to_path, path_to_file_uri +from w3lib.url import file_uri_to_path from zope.interface import implementer -from zope.interface.verify import verifyObject import scrapy from scrapy import Spider, signals -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, FeedExporter, FeedSlot, FileFeedStorage, - FTPFeedStorage, - GCSFeedStorage, IFeedStorage, - S3FeedStorage, - StdoutFeedStorage, ) -from scrapy.settings import Settings -from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler -from tests.mockserver.ftp import MockFTPServer from tests.mockserver.http import MockServer from tests.spiders import ItemSpider from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: - from collections.abc import Callable, Iterable - from os import PathLike + from collections.abc import Awaitable, Callable, Iterable -def path_to_url(path): +def path_to_url(path: str | Path) -> str: return urljoin("file:", pathname2url(str(path))) -def printf_escape(string): - return string.replace("%", "%%") - - -def build_url(path: str | PathLike) -> str: - path_str = str(path) - if path_str[0] != "/": - path_str = "/" + path_str - return urljoin("file:", path_str) - - -def mock_google_cloud_storage() -> tuple[Any, Any, Any]: - """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob - classes and set their proper return values. - """ - from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415 - - client_mock = mock.create_autospec(Client) - - bucket_mock = mock.create_autospec(Bucket) - client_mock.get_bucket.return_value = bucket_mock - - blob_mock = mock.create_autospec(Blob) - bucket_mock.blob.return_value = blob_mock - - return (client_mock, bucket_mock, blob_mock) - - -class TestFileFeedStorage: - def test_store_file_uri(self, tmp_path): - path = tmp_path / "file.txt" - uri = path_to_file_uri(str(path)) - self._assert_stores(FileFeedStorage(uri), path) - - def test_store_file_uri_makedirs(self, tmp_path): - path = tmp_path / "more" / "paths" / "file.txt" - uri = path_to_file_uri(str(path)) - self._assert_stores(FileFeedStorage(uri), path) - - def test_store_direct_path(self, tmp_path): - path = tmp_path / "file.txt" - self._assert_stores(FileFeedStorage(str(path)), path) - - def test_store_direct_path_relative(self, tmp_path): - old_cwd = Path.cwd() - try: - os.chdir(tmp_path) - path = Path("foo", "bar") - self._assert_stores(FileFeedStorage(str(path)), path) - finally: - os.chdir(old_cwd) - - def test_interface(self, tmp_path): - path = tmp_path / "file.txt" - st = FileFeedStorage(str(path)) - verifyObject(IFeedStorage, st) - - @staticmethod - def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None: - storage = FileFeedStorage(str(path), feed_options=feed_options) - spider = scrapy.Spider("default") - file = storage.open(spider) - file.write(b"content") - storage.store(file) - - def test_append(self, tmp_path): - path = tmp_path / "file.txt" - self._store(path) - self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") - - def test_overwrite(self, tmp_path): - path = tmp_path / "file.txt" - self._store(path, {"overwrite": True}) - self._assert_stores( - FileFeedStorage(str(path), feed_options={"overwrite": True}), path - ) - - @staticmethod - def _assert_stores( - storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" - ) -> None: - spider = scrapy.Spider("default") - file = storage.open(spider) - file.write(b"content") - storage.store(file) - assert path.exists() - try: - assert path.read_bytes() == expected_content - finally: - path.unlink() - - def test_preserves_windows_path_without_file_scheme(self): - path = r"C:\Users\user\Desktop\test.txt" - storage = FileFeedStorage(path) - assert storage.path == path - - -@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage -class TestFTPFeedStorage: - def get_test_spider(self, settings=None): - class TestSpider(scrapy.Spider): - name = "test_spider" - - crawler = get_crawler(settings_dict=settings) - return TestSpider.from_crawler(crawler) - - async def _store(self, uri, content, feed_options=None, settings=None): - crawler = get_crawler(settings_dict=settings or {}) - storage = FTPFeedStorage.from_crawler( - crawler, - uri, - feed_options=feed_options, - ) - verifyObject(IFeedStorage, storage) - spider = self.get_test_spider() - file = storage.open(spider) - file.write(content) - await maybe_deferred_to_future(storage.store(file)) - - def _assert_stored(self, path: Path, content): - assert path.exists() - try: - assert path.read_bytes() == content - finally: - path.unlink() - - @coroutine_test - async def test_append(self): - with MockFTPServer() as ftp_server: - filename = "file" - url = ftp_server.url(filename) - feed_options = {"overwrite": False} - await self._store(url, b"foo", feed_options=feed_options) - await self._store(url, b"bar", feed_options=feed_options) - self._assert_stored(ftp_server.path / filename, b"foobar") - - @coroutine_test - async def test_overwrite(self): - with MockFTPServer() as ftp_server: - filename = "file" - url = ftp_server.url(filename) - await self._store(url, b"foo") - await self._store(url, b"bar") - self._assert_stored(ftp_server.path / filename, b"bar") - - @coroutine_test - async def test_append_active_mode(self): - with MockFTPServer() as ftp_server: - settings = {"FEED_STORAGE_FTP_ACTIVE": True} - filename = "file" - url = ftp_server.url(filename) - feed_options = {"overwrite": False} - await self._store(url, b"foo", feed_options=feed_options, settings=settings) - await self._store(url, b"bar", feed_options=feed_options, settings=settings) - self._assert_stored(ftp_server.path / filename, b"foobar") - - @coroutine_test - async def test_overwrite_active_mode(self): - with MockFTPServer() as ftp_server: - settings = {"FEED_STORAGE_FTP_ACTIVE": True} - filename = "file" - url = ftp_server.url(filename) - await self._store(url, b"foo", settings=settings) - await self._store(url, b"bar", settings=settings) - self._assert_stored(ftp_server.path / filename, b"bar") - - def test_uri_auth_quote(self): - # RFC3986: 3.2.1. User Information - pw_quoted = quote(string.punctuation, safe="") - st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) - assert st.password == string.punctuation - - -class MyBlockingFeedStorage(BlockingFeedStorage): - def _store_in_thread(self, file: IO[bytes]) -> None: - return - - -class TestBlockingFeedStorage: - def get_test_spider(self, settings=None): - class TestSpider(scrapy.Spider): - name = "test_spider" - - crawler = get_crawler(settings_dict=settings) - return TestSpider.from_crawler(crawler) - - def test_default_temp_dir(self): - b = MyBlockingFeedStorage() - - storage_file = b.open(self.get_test_spider()) - storage_dir = Path(storage_file.name).parent - assert str(storage_dir) == tempfile.gettempdir() - - def test_temp_file(self, tmp_path): - b = MyBlockingFeedStorage() - - spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) - storage_file = b.open(spider) - storage_dir = Path(storage_file.name).parent - assert storage_dir == tmp_path - - def test_invalid_folder(self, tmp_path): - b = MyBlockingFeedStorage() - - invalid_path = tmp_path / "invalid_path" - spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) - - with pytest.raises(OSError, match="Not a Directory:"): - b.open(spider=spider) - - -@pytest.mark.requires_boto3 -@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage -class TestS3FeedStorage: - def test_parse_credentials(self): - aws_credentials = { - "AWS_ACCESS_KEY_ID": "settings_key", - "AWS_SECRET_ACCESS_KEY": "settings_secret", - "AWS_SESSION_TOKEN": "settings_token", - } - crawler = get_crawler(settings_dict=aws_credentials) - # Instantiate with crawler - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - assert storage.access_key == "settings_key" - assert storage.secret_key == "settings_secret" - assert storage.session_token == "settings_token" - # Instantiate directly - storage = S3FeedStorage( - "s3://mybucket/export.csv", - aws_credentials["AWS_ACCESS_KEY_ID"], - aws_credentials["AWS_SECRET_ACCESS_KEY"], - session_token=aws_credentials["AWS_SESSION_TOKEN"], - ) - assert storage.access_key == "settings_key" - assert storage.secret_key == "settings_secret" - assert storage.session_token == "settings_token" - # URI priority > settings priority - storage = S3FeedStorage( - "s3://uri_key:uri_secret@mybucket/export.csv", - aws_credentials["AWS_ACCESS_KEY_ID"], - aws_credentials["AWS_SECRET_ACCESS_KEY"], - ) - assert storage.access_key == "uri_key" - assert storage.secret_key == "uri_secret" - - @coroutine_test - async def test_store(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - bucket = "mybucket" - key = "export.csv" - storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}") - verifyObject(IFeedStorage, storage) - - file = mock.MagicMock() - - storage.s3_client = mock.MagicMock() - await maybe_deferred_to_future(storage.store(file)) - assert storage.s3_client.upload_fileobj.call_args == mock.call( - Bucket=bucket, Key=key, Fileobj=file - ) - - def test_init_without_acl(self): - storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl is None - - def test_init_with_acl(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl == "custom-acl" - - def test_init_with_endpoint_url(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - endpoint_url="https://example.com", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.endpoint_url == "https://example.com" - - def test_init_with_region_name(self): - region_name = "ap-east-1" - storage = S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - region_name=region_name, - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.region_name == region_name - assert storage.s3_client._client_config.region_name == region_name - - def test_from_crawler_without_acl(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl is None - - def test_without_endpoint_url(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.endpoint_url is None - - def test_without_region_name(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.s3_client._client_config.region_name == "us-east-1" - - def test_from_crawler_with_acl(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "FEED_STORAGE_S3_ACL": "custom-acl", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler( - crawler, - "s3://mybucket/export.csv", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl == "custom-acl" - - def test_from_crawler_with_endpoint_url(self): - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "AWS_ENDPOINT_URL": "https://example.com", - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.endpoint_url == "https://example.com" - - def test_from_crawler_with_region_name(self): - region_name = "ap-east-1" - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "AWS_REGION_NAME": region_name, - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.region_name == region_name - assert storage.s3_client._client_config.region_name == region_name - - @coroutine_test - async def test_store_without_acl(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl is None - - storage.s3_client = mock.MagicMock() - await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) - acl = ( - storage.s3_client.upload_fileobj.call_args[1] - .get("ExtraArgs", {}) - .get("ACL") - ) - assert acl is None - - @coroutine_test - async def test_store_with_acl(self): - storage = S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - assert storage.access_key == "access_key" - assert storage.secret_key == "secret_key" - assert storage.acl == "custom-acl" - - storage.s3_client = mock.MagicMock() - await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) - acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] - assert acl == "custom-acl" - - def test_overwrite_default(self): - with LogCapture() as log: - S3FeedStorage( - "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" - ) - assert "S3 does not support appending to files" not in str(log) - - def test_overwrite_false(self): - with LogCapture() as log: - S3FeedStorage( - "s3://mybucket/export.csv", - "access_key", - "secret_key", - "custom-acl", - feed_options={"overwrite": False}, - ) - assert "S3 does not support appending to files" in str(log) - - -@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage -class TestGCSFeedStorage: - def test_parse_settings(self): - try: - from google.cloud.storage import Client # noqa: F401,PLC0415 - except ImportError: - pytest.skip("GCSFeedStorage requires google-cloud-storage") - - settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} - crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") - assert storage.project_id == "123" - assert storage.acl == "publicRead" - assert storage.bucket_name == "mybucket" - assert storage.blob_name == "export.csv" - - def test_parse_empty_acl(self): - try: - from google.cloud.storage import Client # noqa: F401,PLC0415 - except ImportError: - pytest.skip("GCSFeedStorage requires google-cloud-storage") - - settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} - crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") - assert storage.acl is None - - settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None} - crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") - assert storage.acl is None - - @coroutine_test - async def test_store(self): - try: - from google.cloud.storage import Client # noqa: F401,PLC0415 - except ImportError: - pytest.skip("GCSFeedStorage requires google-cloud-storage") - - uri = "gs://mybucket/export.csv" - project_id = "myproject-123" - acl = "publicRead" - (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() - with mock.patch("google.cloud.storage.Client") as m: - m.return_value = client_mock - - f = mock.Mock() - storage = GCSFeedStorage(uri, project_id, acl) - await maybe_deferred_to_future(storage.store(f)) - - f.seek.assert_called_once_with(0) - m.assert_called_once_with(project=project_id) - client_mock.get_bucket.assert_called_once_with("mybucket") - bucket_mock.blob.assert_called_once_with("export.csv") - blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) - - def test_overwrite_default(self): - with LogCapture() as log: - GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") - assert "GCS does not support appending to files" not in str(log) - - def test_overwrite_false(self): - with LogCapture() as log: - GCSFeedStorage( - "gs://mybucket/export.csv", - "myproject-123", - "custom-acl", - feed_options={"overwrite": False}, - ) - assert "GCS does not support appending to files" in str(log) - - -class TestStdoutFeedStorage: - def test_store(self): - out = BytesIO() - storage = StdoutFeedStorage("stdout:", _stdout=out) - file = storage.open(scrapy.Spider("default")) - file.write(b"content") - storage.store(file) - assert out.getvalue() == b"content" - - def test_overwrite_default(self): - with LogCapture() as log: - StdoutFeedStorage("stdout:") - assert ( - "Standard output (stdout) storage does not support overwriting" - not in str(log) - ) - - def test_overwrite_true(self): - with LogCapture() as log: - StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) - assert "Standard output (stdout) storage does not support overwriting" in str( - log - ) +def printf_escape(s: str) -> str: + return s.replace("%", "%%") class FromCrawlerMixin: @@ -792,8 +238,10 @@ class TestFeedExportBase(ABC): ) -> dict[str, Any]: pass - def _load_until_eof(self, data, load_func): - result = [] + def _load_until_eof( + self, data: bytes, load_func: Callable[[IO[bytes]], Any] + ) -> list[Any]: + result: list[Any] = [] with tempfile.TemporaryFile() as temp: temp.write(data) temp.seek(0) @@ -839,7 +287,7 @@ class IsExportingListener: if self.start_without_finish: self.start_without_finish = False else: - self.finish_before_start = True + self.finish_without_start = True class ExceptionJsonItemExporter(JsonItemExporter): @@ -1050,8 +498,7 @@ class TestFeedExport(TestFeedExportBase): }, } crawler = get_crawler(ItemSpider, settings) - with mock.patch.object(S3FeedStorage, "store"): - yield crawler.crawl(mockserver=self.mockserver) + yield crawler.crawl(mockserver=self.mockserver) assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 @@ -1702,7 +1149,6 @@ class TestFeedExport(TestFeedExportBase): data = await self.exported_no_data(settings) assert data["csv"] == b"" - @pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage @coroutine_test async def test_multiple_feeds_success_logs_blocking_feed_storage(self): settings = { @@ -1834,945 +1280,6 @@ class TestFeedExport(TestFeedExportBase): assert not Storage.file_was_closed -class TestFeedPostProcessedExports(TestFeedExportBase): - items = [{"foo": "bar"}] - expected = b"foo\r\nbar\r\n" - - class MyPlugin1: - def __init__(self, file, feed_options): - self.file = file - self.feed_options = feed_options - self.char = self.feed_options.get("plugin1_char", b"") - - def write(self, data): - written_count = self.file.write(data) - written_count += self.file.write(self.char) - return written_count - - def close(self): - self.file.close() - - def _named_tempfile(self, name) -> str: - return str(Path(self.temp_dir, name)) - - async def run_and_export( - self, spider_cls: type[Spider], settings: dict[str, Any] - ) -> dict[str, bytes | None]: - """Run spider with specified settings; return exported data with filename.""" - - FEEDS = settings.get("FEEDS") or {} - settings["FEEDS"] = { - printf_escape(path_to_url(file_path)): feed_options - for file_path, feed_options in FEEDS.items() - } - - content: dict[str, bytes | None] = {} - try: - spider_cls.start_urls = [self.mockserver.url("/")] - crawler = get_crawler(spider_cls, settings) - await crawler.crawl_async() - - for file_path in FEEDS: - content[str(file_path)] = ( - Path(file_path).read_bytes() if Path(file_path).exists() else None - ) - - finally: - for file_path in FEEDS: - if not Path(file_path).exists(): - continue - - Path(file_path).unlink() - - return content - - def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""): - data_stream = BytesIO() - gzipf = gzip.GzipFile( - fileobj=data_stream, - filename=filename, - mtime=mtime, - compresslevel=compresslevel, - mode="wb", - ) - gzipf.write(data) - gzipf.close() - data_stream.seek(0) - return data_stream.read() - - @coroutine_test - async def test_gzip_plugin(self): - filename = self._named_tempfile("gzip_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - }, - }, - } - - data = await self.exported_data(self.items, settings) - try: - gzip.decompress(data[filename]) - except OSError: - pytest.fail("Received invalid gzip data.") - - @coroutine_test - async def test_gzip_plugin_compresslevel(self): - filename_to_compressed = { - self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( - self.expected, compresslevel=0 - ), - self._named_tempfile("compresslevel_9"): self.get_gzip_compressed( - self.expected, compresslevel=9 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("compresslevel_0"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_compresslevel": 0, - "gzip_mtime": 0, - "gzip_filename": "", - }, - self._named_tempfile("compresslevel_9"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_compresslevel": 9, - "gzip_mtime": 0, - "gzip_filename": "", - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = gzip.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_gzip_plugin_mtime(self): - filename_to_compressed = { - self._named_tempfile("mtime_123"): self.get_gzip_compressed( - self.expected, mtime=123 - ), - self._named_tempfile("mtime_123456789"): self.get_gzip_compressed( - self.expected, mtime=123456789 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("mtime_123"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 123, - "gzip_filename": "", - }, - self._named_tempfile("mtime_123456789"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 123456789, - "gzip_filename": "", - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = gzip.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_gzip_plugin_filename(self): - filename_to_compressed = { - self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( - self.expected, filename="FILE1" - ), - self._named_tempfile("filename_FILE2"): self.get_gzip_compressed( - self.expected, filename="FILE2" - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("filename_FILE1"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 0, - "gzip_filename": "FILE1", - }, - self._named_tempfile("filename_FILE2"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], - "gzip_mtime": 0, - "gzip_filename": "FILE2", - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = gzip.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_lzma_plugin(self): - filename = self._named_tempfile("lzma_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - }, - }, - } - - data = await self.exported_data(self.items, settings) - try: - lzma.decompress(data[filename]) - except lzma.LZMAError: - pytest.fail("Received invalid lzma data.") - - @coroutine_test - async def test_lzma_plugin_format(self): - filename_to_compressed = { - self._named_tempfile("format_FORMAT_XZ"): lzma.compress( - self.expected, format=lzma.FORMAT_XZ - ), - self._named_tempfile("format_FORMAT_ALONE"): lzma.compress( - self.expected, format=lzma.FORMAT_ALONE - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("format_FORMAT_XZ"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_format": lzma.FORMAT_XZ, - }, - self._named_tempfile("format_FORMAT_ALONE"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_format": lzma.FORMAT_ALONE, - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = lzma.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_lzma_plugin_check(self): - filename_to_compressed = { - self._named_tempfile("check_CHECK_NONE"): lzma.compress( - self.expected, check=lzma.CHECK_NONE - ), - self._named_tempfile("check_CHECK_CRC256"): lzma.compress( - self.expected, check=lzma.CHECK_SHA256 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("check_CHECK_NONE"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_check": lzma.CHECK_NONE, - }, - self._named_tempfile("check_CHECK_CRC256"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_check": lzma.CHECK_SHA256, - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = lzma.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_lzma_plugin_preset(self): - filename_to_compressed = { - self._named_tempfile("preset_PRESET_0"): lzma.compress( - self.expected, preset=0 - ), - self._named_tempfile("preset_PRESET_9"): lzma.compress( - self.expected, preset=9 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("preset_PRESET_0"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_preset": 0, - }, - self._named_tempfile("preset_PRESET_9"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_preset": 9, - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = lzma.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_lzma_plugin_filters(self): - if "PyPy" in sys.version: - # https://foss.heptapod.net/pypy/pypy/-/issues/3527 - pytest.skip("lzma filters doesn't work in PyPy") - - filters = [{"id": lzma.FILTER_LZMA2}] - compressed = lzma.compress(self.expected, filters=filters) - filename = self._named_tempfile("filters") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], - "lzma_filters": filters, - }, - }, - } - - data = await self.exported_data(self.items, settings) - assert compressed == data[filename] - result = lzma.decompress(data[filename]) - assert result == self.expected - - @coroutine_test - async def test_bz2_plugin(self): - filename = self._named_tempfile("bz2_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], - }, - }, - } - - data = await self.exported_data(self.items, settings) - try: - bz2.decompress(data[filename]) - except OSError: - pytest.fail("Received invalid bz2 data.") - - @coroutine_test - async def test_bz2_plugin_compresslevel(self): - filename_to_compressed = { - self._named_tempfile("compresslevel_1"): bz2.compress( - self.expected, compresslevel=1 - ), - self._named_tempfile("compresslevel_9"): bz2.compress( - self.expected, compresslevel=9 - ), - } - - settings = { - "FEEDS": { - self._named_tempfile("compresslevel_1"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], - "bz2_compresslevel": 1, - }, - self._named_tempfile("compresslevel_9"): { - "format": "csv", - "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], - "bz2_compresslevel": 9, - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, compressed in filename_to_compressed.items(): - result = bz2.decompress(data[filename]) - assert compressed == data[filename] - assert result == self.expected - - @coroutine_test - async def test_custom_plugin(self): - filename = self._named_tempfile("csv_file") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": [self.MyPlugin1], - }, - }, - } - - data = await self.exported_data(self.items, settings) - assert data[filename] == self.expected - - @coroutine_test - async def test_custom_plugin_with_parameter(self): - expected = b"foo\r\n\nbar\r\n\n" - filename = self._named_tempfile("newline") - - settings = { - "FEEDS": { - filename: { - "format": "csv", - "postprocessing": [self.MyPlugin1], - "plugin1_char": b"\n", - }, - }, - } - - data = await self.exported_data(self.items, settings) - assert data[filename] == expected - - @coroutine_test - async def test_custom_plugin_with_compression(self): - expected = b"foo\r\n\nbar\r\n\n" - - filename_to_decompressor = { - self._named_tempfile("bz2"): bz2.decompress, - self._named_tempfile("lzma"): lzma.decompress, - self._named_tempfile("gzip"): gzip.decompress, - } - - settings = { - "FEEDS": { - self._named_tempfile("bz2"): { - "format": "csv", - "postprocessing": [ - self.MyPlugin1, - "scrapy.extensions.postprocessing.Bz2Plugin", - ], - "plugin1_char": b"\n", - }, - self._named_tempfile("lzma"): { - "format": "csv", - "postprocessing": [ - self.MyPlugin1, - "scrapy.extensions.postprocessing.LZMAPlugin", - ], - "plugin1_char": b"\n", - }, - self._named_tempfile("gzip"): { - "format": "csv", - "postprocessing": [ - self.MyPlugin1, - "scrapy.extensions.postprocessing.GzipPlugin", - ], - "plugin1_char": b"\n", - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, decompressor in filename_to_decompressor.items(): - result = decompressor(data[filename]) - assert result == expected - - @coroutine_test - async def test_exports_compatibility_with_postproc(self): - filename_to_expected = { - self._named_tempfile("csv"): b"foo\r\nbar\r\n", - self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]', - self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n', - self._named_tempfile("xml"): b'\n' - b"\nbar\n", - } - - settings = { - "FEEDS": { - self._named_tempfile("csv"): { - "format": "csv", - "postprocessing": [self.MyPlugin1], - # empty plugin to activate postprocessing.PostProcessingManager - }, - self._named_tempfile("json"): { - "format": "json", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("jsonlines"): { - "format": "jsonlines", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("xml"): { - "format": "xml", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("marshal"): { - "format": "marshal", - "postprocessing": [self.MyPlugin1], - }, - self._named_tempfile("pickle"): { - "format": "pickle", - "postprocessing": [self.MyPlugin1], - }, - }, - } - - data = await self.exported_data(self.items, settings) - - for filename, result in data.items(): - if "pickle" in filename: - expected, result = self.items[0], pickle.loads(result) - elif "marshal" in filename: - expected, result = self.items[0], marshal.loads(result) - else: - expected = filename_to_expected[filename] - assert result == expected - - -class TestBatchDeliveries(TestFeedExportBase): - _file_mark = "_%(batch_time)s_#%(batch_id)02d_" - - async def run_and_export( - self, spider_cls: type[Spider], settings: dict[str, Any] - ) -> dict[str, list[bytes]]: - """Run spider with specified settings; return exported data.""" - - FEEDS = settings.get("FEEDS") or {} - settings["FEEDS"] = { - build_url(file_path): feed for file_path, feed in FEEDS.items() - } - content: defaultdict[str, list[bytes]] = defaultdict(list) - spider_cls.start_urls = [self.mockserver.url("/")] - crawler = get_crawler(spider_cls, settings) - await crawler.crawl_async() - - for path, feed in FEEDS.items(): - dir_name = Path(path).parent - if not dir_name.exists(): - content[feed["format"]] = [] - continue - for file in sorted(dir_name.iterdir()): - content[feed["format"]].append(file.read_bytes()) - return content - - async def assertExportedJsonLines(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "jl" / self._file_mark: { - "format": "jl" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = await self.exported_data(items, settings) - for batch in data["jl"]: - got_batch = [ - json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() - ] - expected_batch, rows = rows[:batch_size], rows[batch_size:] - assert got_batch == expected_batch - - async def assertExportedCsv(self, items, header, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "csv" / self._file_mark: { - "format": "csv" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - data = await self.exported_data(items, settings) - for batch in data["csv"]: - got_batch = csv.DictReader(to_unicode(batch).splitlines()) - assert list(header) == got_batch.fieldnames - expected_batch, rows = rows[:batch_size], rows[batch_size:] - assert list(got_batch) == expected_batch - - async def assertExportedXml(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "xml" / self._file_mark: { - "format": "xml" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = await self.exported_data(items, settings) - for batch in data["xml"]: - root = lxml.etree.fromstring(batch) - got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] - expected_batch, rows = rows[:batch_size], rows[batch_size:] - assert got_batch == expected_batch - - async def assertExportedMultiple(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "xml" / self._file_mark: { - "format": "xml" - }, - self._random_temp_filename() / "json" / self._file_mark: { - "format": "json" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = await self.exported_data(items, settings) - # XML - xml_rows = rows.copy() - for batch in data["xml"]: - root = lxml.etree.fromstring(batch) - got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] - expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] - assert got_batch == expected_batch - # JSON - json_rows = rows.copy() - for batch in data["json"]: - got_batch = json.loads(batch.decode("utf-8")) - expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] - assert got_batch == expected_batch - - async def assertExportedPickle(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "pickle" / self._file_mark: { - "format": "pickle" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = await self.exported_data(items, settings) - - for batch in data["pickle"]: - got_batch = self._load_until_eof(batch, load_func=pickle.load) - expected_batch, rows = rows[:batch_size], rows[batch_size:] - assert got_batch == expected_batch - - async def assertExportedMarshal(self, items, rows, settings=None): - settings = settings or {} - settings.update( - { - "FEEDS": { - self._random_temp_filename() / "marshal" / self._file_mark: { - "format": "marshal" - }, - }, - } - ) - batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") - rows = [{k: v for k, v in row.items() if v} for row in rows] - data = await self.exported_data(items, settings) - - for batch in data["marshal"]: - got_batch = self._load_until_eof(batch, load_func=marshal.load) - expected_batch, rows = rows[:batch_size], rows[batch_size:] - assert got_batch == expected_batch - - @coroutine_test - async def test_export_items(self): - """Test partial deliveries in all supported formats""" - items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), - ] - rows = [ - {"egg": "spam1", "foo": "bar1", "baz": ""}, - {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, - {"foo": "bar3", "baz": "quux3", "egg": ""}, - ] - settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} - header = self.MyItem.fields.keys() - await self.assertExported(items, header, rows, settings=settings) - - def test_wrong_path(self): - """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" - settings = { - "FEEDS": { - self._random_temp_filename(): {"format": "xml"}, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - crawler = get_crawler(settings_dict=settings) - with pytest.raises(NotConfigured): - FeedExporter(crawler) - - @coroutine_test - async def test_export_no_items_not_store_empty(self): - for fmt in ("json", "jsonlines", "xml", "csv"): - settings = { - "FEEDS": { - self._random_temp_filename() / fmt / self._file_mark: { - "format": fmt - }, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - "FEED_STORE_EMPTY": False, - } - data = await self.exported_no_data(settings) - data = dict(data) - assert len(data[fmt]) == 0 - - @coroutine_test - async def test_export_no_items_store_empty(self): - formats = ( - ("json", b"[]"), - ("jsonlines", b""), - ("xml", b'\n'), - ("csv", b""), - ) - - for fmt, expctd in formats: - settings = { - "FEEDS": { - self._random_temp_filename() / fmt / self._file_mark: { - "format": fmt - }, - }, - "FEED_STORE_EMPTY": True, - "FEED_EXPORT_INDENT": None, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - data = await self.exported_no_data(settings) - data = dict(data) - assert data[fmt][0] == expctd - - @coroutine_test - async def test_export_multiple_configs(self): - items = [ - {"foo": "FOO", "bar": "BAR"}, - {"foo": "FOO1", "bar": "BAR1"}, - ] - - formats = { - "json": [ - b'[\n{"bar": "BAR"}\n]', - b'[\n{"bar": "BAR1"}\n]', - ], - "xml": [ - ( - b'\n' - b"\n \n FOO\n \n" - ), - ( - b'\n' - b"\n \n FOO1\n \n" - ), - ], - "csv": [ - b"foo,bar\r\nFOO,BAR\r\n", - b"foo,bar\r\nFOO1,BAR1\r\n", - ], - } - - settings = { - "FEEDS": { - self._random_temp_filename() / "json" / self._file_mark: { - "format": "json", - "indent": 0, - "fields": ["bar"], - "encoding": "utf-8", - }, - self._random_temp_filename() / "xml" / self._file_mark: { - "format": "xml", - "indent": 2, - "fields": ["foo"], - "encoding": "latin-1", - }, - self._random_temp_filename() / "csv" / self._file_mark: { - "format": "csv", - "indent": None, - "fields": ["foo", "bar"], - "encoding": "utf-8", - }, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - data = await self.exported_data(items, settings) - for fmt, expected in formats.items(): - for expected_batch, got_batch in zip(expected, data[fmt], strict=False): - assert got_batch == expected_batch - - @coroutine_test - async def test_batch_item_count_feeds_setting(self): - items = [{"foo": "FOO"}, {"foo": "FOO1"}] - formats = { - "json": [ - b'[{"foo": "FOO"}]', - b'[{"foo": "FOO1"}]', - ], - } - settings = { - "FEEDS": { - self._random_temp_filename() / "json" / self._file_mark: { - "format": "json", - "indent": None, - "encoding": "utf-8", - "batch_item_count": 1, - }, - }, - } - data = await self.exported_data(items, settings) - for fmt, expected in formats.items(): - for expected_batch, got_batch in zip(expected, data[fmt], strict=False): - assert got_batch == expected_batch - - @coroutine_test - async def test_batch_path_differ(self): - """ - Test that the name of all batch files differ from each other. - So %(batch_id)d replaced with the current id. - """ - items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), - ] - settings = { - "FEEDS": { - self._random_temp_filename() / "%(batch_id)d": { - "format": "json", - }, - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - data = await self.exported_data(items, settings) - assert len(items) == len(data["json"]) - - @inline_callbacks_test - def test_stats_batch_file_success(self): - settings = { - "FEEDS": { - build_url( - str(self._random_temp_filename() / "json" / self._file_mark) - ): { - "format": "json", - } - }, - "FEED_EXPORT_BATCH_ITEM_COUNT": 1, - } - crawler = get_crawler(ItemSpider, settings) - yield crawler.crawl(total=2, mockserver=self.mockserver) - assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() - assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 - - @pytest.mark.requires_boto3 - @inline_callbacks_test - def test_s3_export(self): - bucket = "mybucket" - items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), - ] - - class CustomS3FeedStorage(S3FeedStorage): - stubs = [] - - def open(self, *args, **kwargs): - from botocore import __version__ as botocore_version # noqa: PLC0415 - from botocore.stub import ANY, Stubber # noqa: PLC0415 - - expected_params = { - "Body": ANY, - "Bucket": bucket, - "Key": ANY, - } - if Version(botocore_version) >= Version("1.36.0"): - expected_params["ChecksumAlgorithm"] = ANY - - stub = Stubber(self.s3_client) - stub.activate() - CustomS3FeedStorage.stubs.append(stub) - stub.add_response( - "put_object", - expected_params=expected_params, - service_response={}, - ) - return super().open(*args, **kwargs) - - key = "export.csv" - uri = f"s3://{bucket}/{key}/%(batch_id)d.json" - batch_item_count = 1 - settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", - "FEED_EXPORT_BATCH_ITEM_COUNT": batch_item_count, - "FEED_STORAGES": { - "s3": CustomS3FeedStorage, - }, - "FEEDS": { - uri: { - "format": "json", - }, - }, - } - crawler = get_crawler(settings_dict=settings) - storage = S3FeedStorage.from_crawler(crawler, uri) - verifyObject(IFeedStorage, storage) - - class TestSpider(scrapy.Spider): - name = "testspider" - - def parse(self, response): - yield from items - - TestSpider.start_urls = [self.mockserver.url("/")] - crawler = get_crawler(TestSpider, settings) - yield crawler.crawl() - - assert len(CustomS3FeedStorage.stubs) == len(items) - for stub in CustomS3FeedStorage.stubs[:-1]: - stub.assert_no_pending_responses() - assert ( - "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats() - ) - assert ( - crawler.stats.get_value("feedexport/success_count/CustomS3FeedStorage") == 3 - ) - - # Test that the FeedExporer sends the feed_exporter_closed and feed_slot_closed signals class TestFeedExporterSignals: items = [ @@ -2784,7 +1291,7 @@ class TestFeedExporterSignals: with tempfile.NamedTemporaryFile(suffix="json") as tmp: settings = { "FEEDS": { - f"file:///{tmp.name}": { + printf_escape(path_to_url(tmp.name)): { "format": "json", }, }, @@ -2803,7 +1310,9 @@ class TestFeedExporterSignals: self.feed_slot_closed_received = True async def run_signaled_feed_exporter( - self, feed_exporter_signal_handler: Callable, feed_slot_signal_handler: Callable + self, + feed_exporter_signal_handler: Callable[[], Awaitable[None] | None], + feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None], ) -> None: crawler = get_crawler(settings_dict=self.settings) feed_exporter = FeedExporter.from_crawler(crawler) @@ -2893,134 +1402,3 @@ class TestFeedExportInit: crawler = get_crawler(settings_dict=settings) exporter = FeedExporter.from_crawler(crawler) assert isinstance(exporter, FeedExporter) - - -class TestURIParams(ABC): - spider_name = "uri_params_spider" - deprecated_options = False - - @abstractmethod - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - raise NotImplementedError - - def _crawler_feed_exporter(self, settings): - if self.deprecated_options: - with pytest.warns( - ScrapyDeprecationWarning, - match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", - ): - crawler = get_crawler(settings_dict=settings) - else: - crawler = get_crawler(settings_dict=settings) - feed_exporter = crawler.get_extension(FeedExporter) - return crawler, feed_exporter - - def test_default(self): - settings = self.build_settings( - uri="file:///tmp/%(name)s", - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - feed_exporter.open_spider(spider) - - assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" - - def test_none(self): - def uri_params(params, spider): - pass - - settings = self.build_settings( - uri="file:///tmp/%(name)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - - feed_exporter.open_spider(spider) - - assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" - - def test_empty_dict(self): - def uri_params(params, spider): - return {} - - settings = self.build_settings( - uri="file:///tmp/%(name)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - with pytest.raises(KeyError): - feed_exporter.open_spider(spider) - - def test_params_as_is(self): - def uri_params(params, spider): - return params - - settings = self.build_settings( - uri="file:///tmp/%(name)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - feed_exporter.open_spider(spider) - - assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" - - def test_custom_param(self): - def uri_params(params, spider): - return {**params, "foo": self.spider_name} - - settings = self.build_settings( - uri="file:///tmp/%(foo)s", - uri_params=uri_params, - ) - crawler, feed_exporter = self._crawler_feed_exporter(settings) - spider = scrapy.Spider(self.spider_name) - spider.crawler = crawler - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - feed_exporter.open_spider(spider) - - assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" - - -class TestURIParamsSetting(TestURIParams): - deprecated_options = True - - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - extra_settings = {} - if uri_params: - extra_settings["FEED_URI_PARAMS"] = uri_params - return { - "FEED_URI": uri, - **extra_settings, - } - - -class TestURIParamsFeedOption(TestURIParams): - deprecated_options = False - - def build_settings(self, uri="file:///tmp/foobar", uri_params=None): - options = { - "format": "jl", - } - if uri_params: - options["uri_params"] = uri_params - return { - "FEEDS": { - uri: options, - }, - } diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py new file mode 100644 index 000000000..0a926479b --- /dev/null +++ b/tests/test_feedexport_batch.py @@ -0,0 +1,457 @@ +from __future__ import annotations + +import csv +import json +import marshal +import pickle +from collections import defaultdict +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urljoin + +import lxml.etree +import pytest +from packaging.version import Version +from zope.interface.verify import verifyObject + +import scrapy +from scrapy import Spider +from scrapy.exceptions import NotConfigured +from scrapy.extensions.feedexport import FeedExporter, IFeedStorage, S3FeedStorage +from scrapy.settings import Settings +from scrapy.utils.python import to_unicode +from scrapy.utils.test import get_crawler +from tests.spiders import ItemSpider +from tests.test_feedexport import TestFeedExportBase +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + from os import PathLike + + +def build_url(path: str | PathLike[str]) -> str: + path_str = str(path) + if path_str[0] != "/": + path_str = "/" + path_str + return urljoin("file:", path_str) + + +class TestBatchDeliveries(TestFeedExportBase): + _file_mark = "_%(batch_time)s_#%(batch_id)02d_" + + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, list[bytes]]: + """Run spider with specified settings; return exported data.""" + + FEEDS = settings.get("FEEDS") or {} + settings["FEEDS"] = { + build_url(file_path): feed for file_path, feed in FEEDS.items() + } + content: defaultdict[str, list[bytes]] = defaultdict(list) + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + await crawler.crawl_async() + + for path, feed in FEEDS.items(): + dir_name = Path(path).parent + if not dir_name.exists(): + content[feed["format"]] = [] + continue + for file in sorted(dir_name.iterdir()): + content[feed["format"]].append(file.read_bytes()) + return content + + async def assertExportedJsonLines(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "jl" / self._file_mark: { + "format": "jl" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + for batch in data["jl"]: + got_batch = [ + json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines() + ] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedCsv(self, items, header, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "csv" / self._file_mark: { + "format": "csv" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + data = await self.exported_data(items, settings) + for batch in data["csv"]: + got_batch = csv.DictReader(to_unicode(batch).splitlines()) + assert list(header) == got_batch.fieldnames + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert list(got_batch) == expected_batch + + async def assertExportedXml(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + for batch in data["xml"]: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedMultiple(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml" + }, + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + # XML + xml_rows = rows.copy() + for batch in data["xml"]: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall("item")] + expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] + assert got_batch == expected_batch + # JSON + json_rows = rows.copy() + for batch in data["json"]: + got_batch = json.loads(batch.decode("utf-8")) + expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedPickle(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "pickle" / self._file_mark: { + "format": "pickle" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + + for batch in data["pickle"]: + got_batch = self._load_until_eof(batch, load_func=pickle.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + async def assertExportedMarshal(self, items, rows, settings=None): + settings = settings or {} + settings.update( + { + "FEEDS": { + self._random_temp_filename() / "marshal" / self._file_mark: { + "format": "marshal" + }, + }, + } + ) + batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT") + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = await self.exported_data(items, settings) + + for batch in data["marshal"]: + got_batch = self._load_until_eof(batch, load_func=marshal.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + assert got_batch == expected_batch + + @coroutine_test + async def test_export_items(self): + """Test partial deliveries in all supported formats""" + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), + ] + rows = [ + {"egg": "spam1", "foo": "bar1", "baz": ""}, + {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, + {"foo": "bar3", "baz": "quux3", "egg": ""}, + ] + settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} + header = self.MyItem.fields.keys() + await self.assertExported(items, header, rows, settings=settings) + + def test_wrong_path(self): + """If path is without %(batch_time)s and %(batch_id) an exception must be raised""" + settings = { + "FEEDS": { + self._random_temp_filename(): {"format": "xml"}, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + crawler = get_crawler(settings_dict=settings) + with pytest.raises(NotConfigured): + FeedExporter(crawler) + + @coroutine_test + async def test_export_no_items_not_store_empty(self): + for fmt in ("json", "jsonlines", "xml", "csv"): + settings = { + "FEEDS": { + self._random_temp_filename() / fmt / self._file_mark: { + "format": fmt + }, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + "FEED_STORE_EMPTY": False, + } + data = await self.exported_no_data(settings) + data = dict(data) + assert len(data[fmt]) == 0 + + @coroutine_test + async def test_export_no_items_store_empty(self): + formats = ( + ("json", b"[]"), + ("jsonlines", b""), + ("xml", b'\n'), + ("csv", b""), + ) + + for fmt, expctd in formats: + settings = { + "FEEDS": { + self._random_temp_filename() / fmt / self._file_mark: { + "format": fmt + }, + }, + "FEED_STORE_EMPTY": True, + "FEED_EXPORT_INDENT": None, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + data = await self.exported_no_data(settings) + data = dict(data) + assert data[fmt][0] == expctd + + @coroutine_test + async def test_export_multiple_configs(self): + items = [ + {"foo": "FOO", "bar": "BAR"}, + {"foo": "FOO1", "bar": "BAR1"}, + ] + + formats = { + "json": [ + b'[\n{"bar": "BAR"}\n]', + b'[\n{"bar": "BAR1"}\n]', + ], + "xml": [ + ( + b'\n' + b"\n \n FOO\n \n" + ), + ( + b'\n' + b"\n \n FOO1\n \n" + ), + ], + "csv": [ + b"foo,bar\r\nFOO,BAR\r\n", + b"foo,bar\r\nFOO1,BAR1\r\n", + ], + } + + settings = { + "FEEDS": { + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json", + "indent": 0, + "fields": ["bar"], + "encoding": "utf-8", + }, + self._random_temp_filename() / "xml" / self._file_mark: { + "format": "xml", + "indent": 2, + "fields": ["foo"], + "encoding": "latin-1", + }, + self._random_temp_filename() / "csv" / self._file_mark: { + "format": "csv", + "indent": None, + "fields": ["foo", "bar"], + "encoding": "utf-8", + }, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + data = await self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt], strict=True): + assert got_batch == expected_batch + + @coroutine_test + async def test_batch_item_count_feeds_setting(self): + items = [{"foo": "FOO"}, {"foo": "FOO1"}] + formats = { + "json": [ + b'[{"foo": "FOO"}]', + b'[{"foo": "FOO1"}]', + ], + } + settings = { + "FEEDS": { + self._random_temp_filename() / "json" / self._file_mark: { + "format": "json", + "indent": None, + "encoding": "utf-8", + "batch_item_count": 1, + }, + }, + } + data = await self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt], strict=True): + assert got_batch == expected_batch + + @coroutine_test + async def test_batch_path_differ(self): + """ + Test that the name of all batch files differ from each other. + So %(batch_id)d replaced with the current id. + """ + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), + ] + settings = { + "FEEDS": { + self._random_temp_filename() / "%(batch_id)d": { + "format": "json", + }, + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + data = await self.exported_data(items, settings) + assert len(items) == len(data["json"]) + + @inline_callbacks_test + def test_stats_batch_file_success(self): + settings = { + "FEEDS": { + build_url( + str(self._random_temp_filename() / "json" / self._file_mark) + ): { + "format": "json", + } + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + crawler = get_crawler(ItemSpider, settings) + yield crawler.crawl(total=2, mockserver=self.mockserver) + assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() + assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 + + @pytest.mark.requires_boto3 + @inline_callbacks_test + def test_s3_export(self): + bucket = "mybucket" + items = [ + self.MyItem({"foo": "bar1", "egg": "spam1"}), + self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + self.MyItem({"foo": "bar3", "baz": "quux3"}), + ] + + class CustomS3FeedStorage(S3FeedStorage): + stubs = [] + + def open(self, *args, **kwargs): + from botocore import __version__ as botocore_version # noqa: PLC0415 + from botocore.stub import ANY, Stubber # noqa: PLC0415 + + expected_params = { + "Body": ANY, + "Bucket": bucket, + "Key": ANY, + } + if Version(botocore_version) >= Version("1.36.0"): + expected_params["ChecksumAlgorithm"] = ANY + + stub = Stubber(self.s3_client) + stub.activate() + CustomS3FeedStorage.stubs.append(stub) + stub.add_response( + "put_object", + expected_params=expected_params, + service_response={}, + ) + return super().open(*args, **kwargs) + + key = "export.csv" + uri = f"s3://{bucket}/{key}/%(batch_id)d.json" + batch_item_count = 1 + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "FEED_EXPORT_BATCH_ITEM_COUNT": batch_item_count, + "FEED_STORAGES": { + "s3": CustomS3FeedStorage, + }, + "FEEDS": { + uri: { + "format": "json", + }, + }, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, uri) + verifyObject(IFeedStorage, storage) + + class TestSpider(scrapy.Spider): + name = "testspider" + + def parse(self, response): + yield from items + + TestSpider.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(TestSpider, settings) + yield crawler.crawl() + + assert len(CustomS3FeedStorage.stubs) == len(items) + for stub in CustomS3FeedStorage.stubs: + stub.assert_no_pending_responses() + assert ( + "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats() + ) + assert ( + crawler.stats.get_value("feedexport/success_count/CustomS3FeedStorage") == 3 + ) diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py new file mode 100644 index 000000000..6ebcab152 --- /dev/null +++ b/tests/test_feedexport_postprocess.py @@ -0,0 +1,538 @@ +from __future__ import annotations + +import bz2 +import gzip +import lzma +import marshal +import pickle +import sys +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy.utils.test import get_crawler +from tests.test_feedexport import TestFeedExportBase, path_to_url, printf_escape +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy import Spider + + +class TestFeedPostProcessedExports(TestFeedExportBase): + items = [{"foo": "bar"}] + expected = b"foo\r\nbar\r\n" + + class MyPlugin1: + def __init__(self, file, feed_options): + self.file = file + self.feed_options = feed_options + self.char = self.feed_options.get("plugin1_char", b"") + + def write(self, data): + written_count = self.file.write(data) + written_count += self.file.write(self.char) + return written_count + + def close(self): + self.file.close() + + def _named_tempfile(self, name) -> str: + return str(Path(self.temp_dir, name)) + + async def run_and_export( + self, spider_cls: type[Spider], settings: dict[str, Any] + ) -> dict[str, bytes | None]: + """Run spider with specified settings; return exported data with filename.""" + + FEEDS = settings.get("FEEDS") or {} + settings["FEEDS"] = { + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() + } + + content: dict[str, bytes | None] = {} + try: + spider_cls.start_urls = [self.mockserver.url("/")] + crawler = get_crawler(spider_cls, settings) + await crawler.crawl_async() + + for file_path in FEEDS: + content[str(file_path)] = ( + Path(file_path).read_bytes() if Path(file_path).exists() else None + ) + + finally: + for file_path in FEEDS: + if not Path(file_path).exists(): + continue + + Path(file_path).unlink() + + return content + + def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""): + data_stream = BytesIO() + gzipf = gzip.GzipFile( + fileobj=data_stream, + filename=filename, + mtime=mtime, + compresslevel=compresslevel, + mode="wb", + ) + gzipf.write(data) + gzipf.close() + data_stream.seek(0) + return data_stream.read() + + @coroutine_test + async def test_gzip_plugin(self): + filename = self._named_tempfile("gzip_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + }, + }, + } + + data = await self.exported_data(self.items, settings) + try: + gzip.decompress(data[filename]) + except OSError: + pytest.fail("Received invalid gzip data.") + + @coroutine_test + async def test_gzip_plugin_compresslevel(self): + filename_to_compressed = { + self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( + self.expected, compresslevel=0 + ), + self._named_tempfile("compresslevel_9"): self.get_gzip_compressed( + self.expected, compresslevel=9 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("compresslevel_0"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_compresslevel": 0, + "gzip_mtime": 0, + "gzip_filename": "", + }, + self._named_tempfile("compresslevel_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_compresslevel": 9, + "gzip_mtime": 0, + "gzip_filename": "", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_gzip_plugin_mtime(self): + filename_to_compressed = { + self._named_tempfile("mtime_123"): self.get_gzip_compressed( + self.expected, mtime=123 + ), + self._named_tempfile("mtime_123456789"): self.get_gzip_compressed( + self.expected, mtime=123456789 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("mtime_123"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 123, + "gzip_filename": "", + }, + self._named_tempfile("mtime_123456789"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 123456789, + "gzip_filename": "", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_gzip_plugin_filename(self): + filename_to_compressed = { + self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( + self.expected, filename="FILE1" + ), + self._named_tempfile("filename_FILE2"): self.get_gzip_compressed( + self.expected, filename="FILE2" + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("filename_FILE1"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 0, + "gzip_filename": "FILE1", + }, + self._named_tempfile("filename_FILE2"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_mtime": 0, + "gzip_filename": "FILE2", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = gzip.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin(self): + filename = self._named_tempfile("lzma_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + }, + }, + } + + data = await self.exported_data(self.items, settings) + try: + lzma.decompress(data[filename]) + except lzma.LZMAError: + pytest.fail("Received invalid lzma data.") + + @coroutine_test + async def test_lzma_plugin_format(self): + filename_to_compressed = { + self._named_tempfile("format_FORMAT_XZ"): lzma.compress( + self.expected, format=lzma.FORMAT_XZ + ), + self._named_tempfile("format_FORMAT_ALONE"): lzma.compress( + self.expected, format=lzma.FORMAT_ALONE + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("format_FORMAT_XZ"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_format": lzma.FORMAT_XZ, + }, + self._named_tempfile("format_FORMAT_ALONE"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_format": lzma.FORMAT_ALONE, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin_check(self): + filename_to_compressed = { + self._named_tempfile("check_CHECK_NONE"): lzma.compress( + self.expected, check=lzma.CHECK_NONE + ), + self._named_tempfile("CHECK_SHA256"): lzma.compress( + self.expected, check=lzma.CHECK_SHA256 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("check_CHECK_NONE"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_check": lzma.CHECK_NONE, + }, + self._named_tempfile("CHECK_SHA256"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_check": lzma.CHECK_SHA256, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin_preset(self): + filename_to_compressed = { + self._named_tempfile("preset_PRESET_0"): lzma.compress( + self.expected, preset=0 + ), + self._named_tempfile("preset_PRESET_9"): lzma.compress( + self.expected, preset=9 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("preset_PRESET_0"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_preset": 0, + }, + self._named_tempfile("preset_PRESET_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_preset": 9, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = lzma.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_lzma_plugin_filters(self): + if "PyPy" in sys.version: + # https://foss.heptapod.net/pypy/pypy/-/issues/3527 + pytest.skip("lzma filters doesn't work in PyPy") + + filters = [{"id": lzma.FILTER_LZMA2}] + compressed = lzma.compress(self.expected, filters=filters) + filename = self._named_tempfile("filters") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"], + "lzma_filters": filters, + }, + }, + } + + data = await self.exported_data(self.items, settings) + assert compressed == data[filename] + result = lzma.decompress(data[filename]) + assert result == self.expected + + @coroutine_test + async def test_bz2_plugin(self): + filename = self._named_tempfile("bz2_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + }, + }, + } + + data = await self.exported_data(self.items, settings) + try: + bz2.decompress(data[filename]) + except OSError: + pytest.fail("Received invalid bz2 data.") + + @coroutine_test + async def test_bz2_plugin_compresslevel(self): + filename_to_compressed = { + self._named_tempfile("compresslevel_1"): bz2.compress( + self.expected, compresslevel=1 + ), + self._named_tempfile("compresslevel_9"): bz2.compress( + self.expected, compresslevel=9 + ), + } + + settings = { + "FEEDS": { + self._named_tempfile("compresslevel_1"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + "bz2_compresslevel": 1, + }, + self._named_tempfile("compresslevel_9"): { + "format": "csv", + "postprocessing": ["scrapy.extensions.postprocessing.Bz2Plugin"], + "bz2_compresslevel": 9, + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, compressed in filename_to_compressed.items(): + result = bz2.decompress(data[filename]) + assert compressed == data[filename] + assert result == self.expected + + @coroutine_test + async def test_custom_plugin(self): + filename = self._named_tempfile("csv_file") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": [self.MyPlugin1], + }, + }, + } + + data = await self.exported_data(self.items, settings) + assert data[filename] == self.expected + + @coroutine_test + async def test_custom_plugin_with_parameter(self): + expected = b"foo\r\n\nbar\r\n\n" + filename = self._named_tempfile("newline") + + settings = { + "FEEDS": { + filename: { + "format": "csv", + "postprocessing": [self.MyPlugin1], + "plugin1_char": b"\n", + }, + }, + } + + data = await self.exported_data(self.items, settings) + assert data[filename] == expected + + @coroutine_test + async def test_custom_plugin_with_compression(self): + expected = b"foo\r\n\nbar\r\n\n" + + filename_to_decompressor = { + self._named_tempfile("bz2"): bz2.decompress, + self._named_tempfile("lzma"): lzma.decompress, + self._named_tempfile("gzip"): gzip.decompress, + } + + settings = { + "FEEDS": { + self._named_tempfile("bz2"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.Bz2Plugin", + ], + "plugin1_char": b"\n", + }, + self._named_tempfile("lzma"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.LZMAPlugin", + ], + "plugin1_char": b"\n", + }, + self._named_tempfile("gzip"): { + "format": "csv", + "postprocessing": [ + self.MyPlugin1, + "scrapy.extensions.postprocessing.GzipPlugin", + ], + "plugin1_char": b"\n", + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, decompressor in filename_to_decompressor.items(): + result = decompressor(data[filename]) + assert result == expected + + @coroutine_test + async def test_exports_compatibility_with_postproc(self): + filename_to_expected = { + self._named_tempfile("csv"): b"foo\r\nbar\r\n", + self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]', + self._named_tempfile("jsonlines"): b'{"foo": "bar"}\n', + self._named_tempfile("xml"): b'\n' + b"\nbar\n", + } + + settings = { + "FEEDS": { + self._named_tempfile("csv"): { + "format": "csv", + "postprocessing": [self.MyPlugin1], + # empty plugin to activate postprocessing.PostProcessingManager + }, + self._named_tempfile("json"): { + "format": "json", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("jsonlines"): { + "format": "jsonlines", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("xml"): { + "format": "xml", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("marshal"): { + "format": "marshal", + "postprocessing": [self.MyPlugin1], + }, + self._named_tempfile("pickle"): { + "format": "pickle", + "postprocessing": [self.MyPlugin1], + }, + }, + } + + data = await self.exported_data(self.items, settings) + + for filename, result in data.items(): + if "pickle" in filename: + expected, result = self.items[0], pickle.loads(result) + elif "marshal" in filename: + expected, result = self.items[0], marshal.loads(result) + else: + expected = filename_to_expected[filename] + assert result == expected diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py new file mode 100644 index 000000000..11e79775f --- /dev/null +++ b/tests/test_feedexport_storages.py @@ -0,0 +1,583 @@ +from __future__ import annotations + +import os +import string +import tempfile +from io import BytesIO +from pathlib import Path +from typing import IO, Any +from unittest import mock +from urllib.parse import quote + +import pytest +from testfixtures import LogCapture +from w3lib.url import path_to_file_uri +from zope.interface.verify import verifyObject + +import scrapy +from scrapy.extensions.feedexport import ( + BlockingFeedStorage, + FileFeedStorage, + FTPFeedStorage, + GCSFeedStorage, + IFeedStorage, + S3FeedStorage, + StdoutFeedStorage, +) +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.test import get_crawler +from tests.mockserver.ftp import MockFTPServer +from tests.utils.decorators import coroutine_test + + +def mock_google_cloud_storage() -> tuple[Any, Any, Any]: + """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob + classes and set their proper return values. + """ + from google.cloud.storage import Blob, Bucket, Client # noqa: PLC0415 + + client_mock = mock.create_autospec(Client) + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + return (client_mock, bucket_mock, blob_mock) + + +class TestFileFeedStorage: + def test_store_file_uri(self, tmp_path): + path = tmp_path / "file.txt" + uri = path_to_file_uri(str(path)) + self._assert_stores(FileFeedStorage(uri), path) + + def test_store_file_uri_makedirs(self, tmp_path): + path = tmp_path / "more" / "paths" / "file.txt" + uri = path_to_file_uri(str(path)) + self._assert_stores(FileFeedStorage(uri), path) + + def test_store_direct_path(self, tmp_path): + path = tmp_path / "file.txt" + self._assert_stores(FileFeedStorage(str(path)), path) + + def test_store_direct_path_relative(self, tmp_path): + old_cwd = Path.cwd() + try: + os.chdir(tmp_path) + path = Path("foo", "bar") + self._assert_stores(FileFeedStorage(str(path)), path) + finally: + os.chdir(old_cwd) + + def test_interface(self, tmp_path): + path = tmp_path / "file.txt" + st = FileFeedStorage(str(path)) + verifyObject(IFeedStorage, st) + + @staticmethod + def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None: + storage = FileFeedStorage(str(path), feed_options=feed_options) + spider = scrapy.Spider("default") + file = storage.open(spider) + file.write(b"content") + storage.store(file) + + def test_append(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path) + self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") + + def test_overwrite(self, tmp_path): + path = tmp_path / "file.txt" + self._store(path, {"overwrite": True}) + self._assert_stores( + FileFeedStorage(str(path), feed_options={"overwrite": True}), path + ) + + @staticmethod + def _assert_stores( + storage: FileFeedStorage, path: Path, expected_content: bytes = b"content" + ) -> None: + spider = scrapy.Spider("default") + file = storage.open(spider) + file.write(b"content") + storage.store(file) + assert path.exists() + try: + assert path.read_bytes() == expected_content + finally: + path.unlink() + + def test_preserves_windows_path_without_file_scheme(self): + path = r"C:\Users\user\Desktop\test.txt" + storage = FileFeedStorage(path) + assert storage.path == path + + +class TestFTPFeedStorage: + def get_test_spider(self, settings=None): + class TestSpider(scrapy.Spider): + name = "test_spider" + + crawler = get_crawler(settings_dict=settings) + return TestSpider.from_crawler(crawler) + + async def _store(self, uri, content, feed_options=None, settings=None): + crawler = get_crawler(settings_dict=settings or {}) + storage = FTPFeedStorage.from_crawler( + crawler, + uri, + feed_options=feed_options, + ) + verifyObject(IFeedStorage, storage) + spider = self.get_test_spider() + file = storage.open(spider) + file.write(content) + await maybe_deferred_to_future(storage.store(file)) + + def _assert_stored(self, path: Path, content): + assert path.exists() + try: + assert path.read_bytes() == content + finally: + path.unlink() + + @coroutine_test + async def test_append(self): + with MockFTPServer() as ftp_server: + filename = "file" + url = ftp_server.url(filename) + feed_options = {"overwrite": False} + await self._store(url, b"foo", feed_options=feed_options) + await self._store(url, b"bar", feed_options=feed_options) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @coroutine_test + async def test_overwrite(self): + with MockFTPServer() as ftp_server: + filename = "file" + url = ftp_server.url(filename) + await self._store(url, b"foo") + await self._store(url, b"bar") + self._assert_stored(ftp_server.path / filename, b"bar") + + @coroutine_test + async def test_append_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {"FEED_STORAGE_FTP_ACTIVE": True} + filename = "file" + url = ftp_server.url(filename) + feed_options = {"overwrite": False} + await self._store(url, b"foo", feed_options=feed_options, settings=settings) + await self._store(url, b"bar", feed_options=feed_options, settings=settings) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @coroutine_test + async def test_overwrite_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {"FEED_STORAGE_FTP_ACTIVE": True} + filename = "file" + url = ftp_server.url(filename) + await self._store(url, b"foo", settings=settings) + await self._store(url, b"bar", settings=settings) + self._assert_stored(ftp_server.path / filename, b"bar") + + def test_uri_auth_quote(self): + # RFC3986: 3.2.1. User Information + pw_quoted = quote(string.punctuation, safe="") + st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {}) + assert st.password == string.punctuation + + +class MyBlockingFeedStorage(BlockingFeedStorage): + def _store_in_thread(self, file: IO[bytes]) -> None: + return + + +class TestBlockingFeedStorage: + def get_test_spider(self, settings=None): + class TestSpider(scrapy.Spider): + name = "test_spider" + + crawler = get_crawler(settings_dict=settings) + return TestSpider.from_crawler(crawler) + + def test_default_temp_dir(self): + b = MyBlockingFeedStorage() + + storage_file = b.open(self.get_test_spider()) + storage_dir = Path(storage_file.name).parent + assert str(storage_dir) == tempfile.gettempdir() + + def test_temp_file(self, tmp_path): + b = MyBlockingFeedStorage() + + spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) + storage_file = b.open(spider) + storage_dir = Path(storage_file.name).parent + assert storage_dir == tmp_path + + def test_invalid_folder(self, tmp_path): + b = MyBlockingFeedStorage() + + invalid_path = tmp_path / "invalid_path" + spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) + + with pytest.raises(OSError, match="Not a Directory:"): + b.open(spider=spider) + + +@pytest.mark.requires_boto3 +class TestS3FeedStorage: + def test_parse_credentials(self): + aws_credentials = { + "AWS_ACCESS_KEY_ID": "settings_key", + "AWS_SECRET_ACCESS_KEY": "settings_secret", + "AWS_SESSION_TOKEN": "settings_token", + } + crawler = get_crawler(settings_dict=aws_credentials) + # Instantiate with crawler + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "settings_key" + assert storage.secret_key == "settings_secret" + assert storage.session_token == "settings_token" + # Instantiate directly + storage = S3FeedStorage( + "s3://mybucket/export.csv", + aws_credentials["AWS_ACCESS_KEY_ID"], + aws_credentials["AWS_SECRET_ACCESS_KEY"], + session_token=aws_credentials["AWS_SESSION_TOKEN"], + ) + assert storage.access_key == "settings_key" + assert storage.secret_key == "settings_secret" + assert storage.session_token == "settings_token" + # URI priority > settings priority + storage = S3FeedStorage( + "s3://uri_key:uri_secret@mybucket/export.csv", + aws_credentials["AWS_ACCESS_KEY_ID"], + aws_credentials["AWS_SECRET_ACCESS_KEY"], + ) + assert storage.access_key == "uri_key" + assert storage.secret_key == "uri_secret" + + @coroutine_test + async def test_store(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + bucket = "mybucket" + key = "export.csv" + storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}") + verifyObject(IFeedStorage, storage) + + file = mock.MagicMock() + + storage.s3_client = mock.MagicMock() + await maybe_deferred_to_future(storage.store(file)) + assert storage.s3_client.upload_fileobj.call_args == mock.call( + Bucket=bucket, Key=key, Fileobj=file + ) + + def test_init_without_acl(self): + storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None + + def test_init_with_acl(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" + + def test_init_with_endpoint_url(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + endpoint_url="https://example.com", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url == "https://example.com" + + def test_init_with_region_name(self): + region_name = "ap-east-1" + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + region_name=region_name, + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name + + def test_from_crawler_without_acl(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None + + def test_without_endpoint_url(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url is None + + def test_without_region_name(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.s3_client._client_config.region_name == "us-east-1" + + def test_from_crawler_with_acl(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "FEED_STORAGE_S3_ACL": "custom-acl", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" + + def test_from_crawler_with_endpoint_url(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_ENDPOINT_URL": "https://example.com", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.endpoint_url == "https://example.com" + + def test_from_crawler_with_region_name(self): + region_name = "ap-east-1" + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_REGION_NAME": region_name, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.region_name == region_name + assert storage.s3_client._client_config.region_name == region_name + + @coroutine_test + async def test_store_without_acl(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl is None + + storage.s3_client = mock.MagicMock() + await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) + acl = ( + storage.s3_client.upload_fileobj.call_args[1] + .get("ExtraArgs", {}) + .get("ACL") + ) + assert acl is None + + @coroutine_test + async def test_store_with_acl(self): + storage = S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert storage.access_key == "access_key" + assert storage.secret_key == "secret_key" + assert storage.acl == "custom-acl" + + storage.s3_client = mock.MagicMock() + await maybe_deferred_to_future(storage.store(BytesIO(b"test file"))) + acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] + assert acl == "custom-acl" + + def test_overwrite_default(self): + with LogCapture() as log: + S3FeedStorage( + "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" + ) + assert "S3 does not support appending to files" not in str(log) + + def test_overwrite_false(self): + with LogCapture() as log: + S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + "custom-acl", + feed_options={"overwrite": False}, + ) + assert "S3 does not support appending to files" in str(log) + + +class TestGCSFeedStorage: + def test_parse_settings(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") + assert storage.project_id == "123" + assert storage.acl == "publicRead" + assert storage.bucket_name == "mybucket" + assert storage.blob_name == "export.csv" + + def test_parse_empty_acl(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") + assert storage.acl is None + + settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None} + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") + assert storage.acl is None + + @coroutine_test + async def test_store(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + uri = "gs://mybucket/export.csv" + project_id = "myproject-123" + acl = "publicRead" + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + with mock.patch("google.cloud.storage.Client") as m: + m.return_value = client_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id, acl) + await maybe_deferred_to_future(storage.store(f)) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with("mybucket") + bucket_mock.blob.assert_called_once_with("export.csv") + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + f.close.assert_called_once_with() + + @coroutine_test + async def test_store_closes_file_on_upload_error(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + uri = "gs://mybucket/export.csv" + project_id = "myproject-123" + acl = "publicRead" + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + blob_mock.upload_from_file.side_effect = OSError("Upload failed") + with mock.patch("google.cloud.storage.Client") as m: + m.return_value = client_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id, acl) + with pytest.raises(OSError, match="Upload failed"): + await maybe_deferred_to_future(storage.store(f)) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with("mybucket") + bucket_mock.blob.assert_called_once_with("export.csv") + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + f.close.assert_called_once_with() + + def test_overwrite_default(self): + with LogCapture() as log: + GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl") + assert "GCS does not support appending to files" not in str(log) + + def test_overwrite_false(self): + with LogCapture() as log: + GCSFeedStorage( + "gs://mybucket/export.csv", + "myproject-123", + "custom-acl", + feed_options={"overwrite": False}, + ) + assert "GCS does not support appending to files" in str(log) + + +class TestStdoutFeedStorage: + def test_store(self): + out = BytesIO() + storage = StdoutFeedStorage("stdout:", _stdout=out) + file = storage.open(scrapy.Spider("default")) + file.write(b"content") + storage.store(file) + assert out.getvalue() == b"content" + + def test_overwrite_default(self): + with LogCapture() as log: + StdoutFeedStorage("stdout:") + assert ( + "Standard output (stdout) storage does not support overwriting" + not in str(log) + ) + + def test_overwrite_true(self): + with LogCapture() as log: + StdoutFeedStorage("stdout:", feed_options={"overwrite": True}) + assert "Standard output (stdout) storage does not support overwriting" in str( + log + ) diff --git a/tests/test_feedexport_uri_params.py b/tests/test_feedexport_uri_params.py new file mode 100644 index 000000000..150d8449f --- /dev/null +++ b/tests/test_feedexport_uri_params.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import warnings +from abc import ABC, abstractmethod + +import pytest + +import scrapy +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.extensions.feedexport import FeedExporter +from scrapy.utils.test import get_crawler + + +class TestURIParams(ABC): + spider_name = "uri_params_spider" + deprecated_options = False + + @abstractmethod + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): + raise NotImplementedError + + def _crawler_feed_exporter(self, settings): + if self.deprecated_options: + with pytest.warns( + ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated", + ): + crawler = get_crawler(settings_dict=settings) + else: + crawler = get_crawler(settings_dict=settings) + feed_exporter = crawler.get_extension(FeedExporter) + return crawler, feed_exporter + + def test_default(self): + settings = self.build_settings( + uri="file:///tmp/%(name)s", + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + def test_none(self): + def uri_params(params, spider): + pass + + settings = self.build_settings( + uri="file:///tmp/%(name)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + def test_empty_dict(self): + def uri_params(params, spider): + return {} + + settings = self.build_settings( + uri="file:///tmp/%(name)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + with pytest.raises(KeyError): + feed_exporter.open_spider(spider) + + def test_params_as_is(self): + def uri_params(params, spider): + return params + + settings = self.build_settings( + uri="file:///tmp/%(name)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + def test_custom_param(self): + def uri_params(params, spider): + return {**params, "foo": self.spider_name} + + settings = self.build_settings( + uri="file:///tmp/%(foo)s", + uri_params=uri_params, + ) + crawler, feed_exporter = self._crawler_feed_exporter(settings) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + feed_exporter.open_spider(spider) + + assert feed_exporter.slots[0].uri == f"file:///tmp/{self.spider_name}" + + +class TestURIParamsSetting(TestURIParams): + deprecated_options = True + + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): + extra_settings = {} + if uri_params: + extra_settings["FEED_URI_PARAMS"] = uri_params + return { + "FEED_URI": uri, + **extra_settings, + } + + +class TestURIParamsFeedOption(TestURIParams): + deprecated_options = False + + def build_settings(self, uri="file:///tmp/foobar", uri_params=None): + options = { + "format": "jl", + } + if uri_params: + options["uri_params"] = uri_params + return { + "FEEDS": { + uri: options, + }, + } diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index af9941f3b..3c1347fd3 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -40,7 +40,7 @@ if TYPE_CHECKING: pytestmark = [ - pytest.mark.requires_reactor, + pytest.mark.requires_reactor, # H2ClientProtocol requires a reactor pytest.mark.skipif( not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" ), @@ -61,7 +61,7 @@ def make_html_body(val: str) -> bytes: class DummySpider(Spider): name = "dummy" - start_urls: list = [] + start_urls = [] def parse(self, response): print(response) @@ -141,7 +141,7 @@ class Dataloss(LeafResource): class NoContentLengthHeader(LeafResource): def render_GET(self, request: TxRequest): - request.requestHeaders.removeHeader("Content-Length") + request.responseHeaders.removeHeader("Content-Length") self.deferRequest(request, 0, self._delayed_render, request) return NOT_DONE_YET @@ -234,7 +234,7 @@ class TestHttps2ClientProtocol: pem = self.key_file.read_text( encoding="utf-8" ) + self.certificate_file.read_text(encoding="utf-8") - return PrivateCertificate.loadPEM(pem) + return PrivateCertificate.loadPEM(pem) # type: ignore[no-any-return] @async_yield_fixture # type: ignore[untyped-decorator] async def client( @@ -411,9 +411,22 @@ class TestHttps2ClientProtocol: client, request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200 ) - async def _check_POST_json_x10(self, *args, **kwargs): + async def _check_POST_json_x10( + self, + client: H2ClientProtocol, + request: Request, + expected_request_body: dict[str, str], + expected_extra_data: str, + expected_status: int, + ) -> None: async def get_coro() -> None: - await self._check_POST_json(*args, **kwargs) + await self._check_POST_json( + client, + request, + expected_request_body, + expected_extra_data, + expected_status, + ) await self._check_repeat(get_coro, 10) @@ -447,9 +460,7 @@ class TestHttps2ClientProtocol: def test_invalid_negotiated_protocol( self, server_port: int, client: H2ClientProtocol ) -> Generator[Deferred[Any], Any, None]: - with mock.patch( - "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2" - ): + with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", new=b"not-h2"): request = Request(url=self.get_url(server_port, "/status?n=200")) with pytest.raises(ResponseFailed): yield make_request_dfd(client, request) @@ -658,6 +669,9 @@ class TestHttps2ClientProtocol: response = await make_request(client, request) assert response.status == status + @pytest.mark.filterwarnings( + r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning" + ) @deferred_f_from_coro_f async def test_response_has_correct_certificate_ip_address( self, @@ -717,7 +731,7 @@ class TestHttps2ClientProtocol: request = Request(self.get_url(server_port, "/timeout")) # Update the timer to 1s to test connection timeout - client.setTimeout(1) + client.setTimeout(1) # type: ignore[no-untyped-call] with pytest.raises(ResponseFailed) as exc_info: yield make_request_dfd(client, request) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 93cbcd1eb..e22689d49 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,24 +1,12 @@ -import json -import re import warnings import xmlrpc.client from typing import Any -from unittest import mock -from urllib.parse import parse_qs, unquote_to_bytes import pytest -from scrapy.http import ( - FormRequest, - Headers, - HtmlResponse, - JsonRequest, - Request, - XmlRpcRequest, -) +from scrapy.http import Headers, Request, XmlRpcRequest from scrapy.http.request import NO_CALLBACK -from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.python import to_bytes class TestRequest: @@ -35,7 +23,6 @@ class TestRequest: # url argument must be basestring with pytest.raises(TypeError): self.request_class(123) - r = self.request_class("http://www.example.com") r = self.request_class("http://www.example.com") assert isinstance(r.url, str) @@ -174,6 +161,19 @@ class TestRequest: r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") assert r4.url == "http://www.example.org/r%E9sum%E9.html" + def test_url_verbatim(self): + r = self.request_class( + url="http://www.scrapy.org/price/£", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/price/£" + + r = self.request_class( + url="http://www.scrapy.org/blank space", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/blank space" + def test_body(self): r1 = self.request_class(url="http://www.example.com/") assert r1.body == b"" @@ -210,11 +210,11 @@ class TestRequest: r1.cb_kwargs["key"] = "value" r2 = r1.copy() - # make sure copy does not propagate callbacks + # make sure callbaclks are copied assert r1.callback is somecallback assert r1.errback is somecallback assert r2.callback is r1.callback - assert r2.errback is r2.errback + assert r2.errback is r1.errback # make sure flags list is shallow copied assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical" @@ -332,12 +332,6 @@ class TestRequest: def test_setters(self): request = self.request_class("http://example.com") - request.cb_kwargs = {"a": 1} - assert request.cb_kwargs == {"a": 1} - - request.meta = {"k": "v"} - assert request.meta == {"k": "v"} - request.flags = ["f1"] assert request.flags == ["f1"] @@ -361,30 +355,13 @@ class TestRequest: request = self.request_class("http://example.com") - assert request._cb_kwargs is None - assert request.cb_kwargs == {} - assert request.cb_kwargs is request.cb_kwargs - assert request._cb_kwargs == {} - original_cb_kwargs = request.cb_kwargs - request.cb_kwargs = None - assert request.cb_kwargs == {} - assert request.cb_kwargs is not original_cb_kwargs - - assert request._meta is None - assert request.meta == {} - assert request.meta is request.meta - assert request._meta == {} - original_meta = request.meta - request.meta = None - assert request.meta == {} - assert request.meta is not original_meta - assert request._flags is None assert request.flags == [] assert request.flags is request.flags assert request._flags == [] original_flags = request.flags request.flags = None + assert request._flags is None assert request.flags == [] assert request.flags is not original_flags @@ -394,6 +371,7 @@ class TestRequest: assert request._cookies == {} original_cookies = request.cookies request.cookies = None + assert request._cookies is None assert request.cookies == {} assert request.cookies is not original_cookies @@ -409,7 +387,9 @@ class TestRequest: assert isinstance(request._headers, Headers) original_headers = request.headers request.headers = None + assert request._headers is None assert request.headers == {} + assert request._headers == {} assert request.headers is not original_headers def test_no_callback(self): @@ -480,11 +460,13 @@ class TestRequest: def test_from_curl_ignore_unknown_options(self): # By default: it works and ignores the unknown options: --foo and -z with warnings.catch_warnings(): # avoid warning when executing tests - warnings.simplefilter("ignore") + warnings.filterwarnings( + "ignore", category=UserWarning, message="Unrecognized options:" + ) r = self.request_class.from_curl( 'curl -X DELETE "http://example.org" --foo -z', ) - assert r.method == "DELETE" + assert r.method == "DELETE" # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z @@ -495,1022 +477,6 @@ class TestRequest: ) -class TestFormRequest(TestRequest): - request_class = FormRequest - - def assertQueryEqual(self, first, second, msg=None): - first = to_unicode(first).split("&") - second = to_unicode(second).split("&") - assert sorted(first) == sorted(second), msg - - def test_empty_formdata(self): - r1 = self.request_class("http://www.example.com", formdata={}) - assert r1.body == b"" - - def test_formdata_overrides_querystring(self): - data = (("a", "one"), ("a", "two"), ("b", "2")) - url = self.request_class( - "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data - ).url.split("#", maxsplit=1)[0] - fs = _qs(self.request_class(url, method="GET", formdata=data)) - assert set(fs[b"a"]) == {b"one", b"two"} - assert fs[b"b"] == [b"2"] - assert fs.get(b"c") is None - - data = {"a": "1", "b": "2"} - fs = _qs( - self.request_class("http://www.example.com/", method="GET", formdata=data) - ) - assert fs[b"a"] == [b"1"] - assert fs[b"b"] == [b"2"] - - def test_default_encoding_bytes(self): - # using default encoding (utf-8) - data = {b"one": b"two", b"price": b"\xc2\xa3 100"} - r2 = self.request_class("http://www.example.com", formdata=data) - assert r2.method == "POST" - assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") - assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" - - def test_default_encoding_textual_data(self): - # using default encoding (utf-8) - data = {"µ one": "two", "price": "£ 100"} - r2 = self.request_class("http://www.example.com", formdata=data) - assert r2.method == "POST" - assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") - assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" - - def test_default_encoding_mixed_data(self): - # using default encoding (utf-8) - data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} - r2 = self.request_class("http://www.example.com", formdata=data) - assert r2.method == "POST" - assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") - assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" - - def test_custom_encoding_bytes(self): - data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} - r2 = self.request_class( - "http://www.example.com", formdata=data, encoding="latin1" - ) - assert r2.method == "POST" - assert r2.encoding == "latin1" - self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") - assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" - - def test_custom_encoding_textual_data(self): - data = {"price": "£ 100"} - r3 = self.request_class( - "http://www.example.com", formdata=data, encoding="latin1" - ) - assert r3.encoding == "latin1" - assert r3.body == b"price=%A3+100" - - def test_multi_key_values(self): - # using multiples values for a single key - data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} - r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual( - r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" - ) - - def test_from_response_post(self): - response = _buildresponse( - b""" - - - - """, - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - assert req.method == "POST" - assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req) - assert set(fs[b"test"]) == {b"val1", b"val2"} - assert set(fs[b"one"]) == {b"two", b"three"} - assert fs[b"test2"] == [b"xxx"] - assert fs[b"six"] == [b"seven"] - - def test_from_response_post_nonascii_bytes_utf8(self): - response = _buildresponse( - b"""
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - assert req.method == "POST" - assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) - assert set(fs["test £"]) == {"val1", "val2"} - assert set(fs["one"]) == {"two", "three"} - assert fs["test2"] == ["xxx µ"] - assert fs["six"] == ["seven"] - - def test_from_response_post_nonascii_bytes_latin1(self): - response = _buildresponse( - b"""
- - - -
""", - url="http://www.example.com/this/list.html", - encoding="latin1", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - assert req.method == "POST" - assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True, encoding="latin1") - assert set(fs["test £"]) == {"val1", "val2"} - assert set(fs["one"]) == {"two", "three"} - assert fs["test2"] == ["xxx µ"] - assert fs["six"] == ["seven"] - - def test_from_response_post_nonascii_unicode(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - - assert req.method == "POST" - assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) - assert set(fs["test £"]) == {"val1", "val2"} - assert set(fs["one"]) == {"two", "three"} - assert fs["test2"] == ["xxx µ"] - assert fs["six"] == ["seven"] - - def test_from_response_duplicate_form_key(self): - response = _buildresponse("
", url="http://www.example.com") - req = self.request_class.from_response( - response=response, - method="GET", - formdata=(("foo", "bar"), ("foo", "baz")), - ) - assert urlparse_cached(req).hostname == "www.example.com" - assert urlparse_cached(req).query == "foo=bar&foo=baz" - - def test_from_response_override_duplicate_form_key(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response( - response, formdata=(("two", "2"), ("two", "4")) - ) - fs = _qs(req) - assert fs[b"one"] == [b"1"] - assert fs[b"two"] == [b"2", b"4"] - - def test_from_response_extra_headers(self): - response = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response( - response=response, - formdata={"one": ["two", "three"], "six": "seven"}, - headers={"Accept-Encoding": "gzip,deflate"}, - ) - assert req.method == "POST" - assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" - assert req.headers["Accept-Encoding"] == b"gzip,deflate" - - def test_from_response_get(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - r1 = self.request_class.from_response( - response, formdata={"one": ["two", "three"], "six": "seven"} - ) - assert r1.method == "GET" - assert urlparse_cached(r1).hostname == "www.example.com" - assert urlparse_cached(r1).path == "/this/get.php" - fs = _qs(r1) - assert set(fs[b"test"]) == {b"val1", b"val2"} - assert set(fs[b"one"]) == {b"two", b"three"} - assert fs[b"test2"] == [b"xxx"] - assert fs[b"six"] == [b"seven"] - - def test_from_response_override_params(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response(response, formdata={"two": "2"}) - fs = _qs(req) - assert fs[b"one"] == [b"1"] - assert fs[b"two"] == [b"2"] - - def test_from_response_drop_params(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response(response, formdata={"two": None}) - fs = _qs(req) - assert fs[b"one"] == [b"1"] - assert b"two" not in fs - - def test_from_response_override_method(self): - response = _buildresponse( - """ -
- """ - ) - request = FormRequest.from_response(response) - assert request.method == "GET" - request = FormRequest.from_response(response, method="POST") - assert request.method == "POST" - - def test_from_response_override_url(self): - response = _buildresponse( - """ -
- """ - ) - request = FormRequest.from_response(response) - assert request.url == "http://example.com/app" - request = FormRequest.from_response(response, url="http://foo.bar/absolute") - assert request.url == "http://foo.bar/absolute" - request = FormRequest.from_response(response, url="/relative") - assert request.url == "http://example.com/relative" - - def test_from_response_case_insensitive(self): - response = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response(response) - fs = _qs(req) - assert fs[b"clickable1"] == [b"clicked1"] - assert b"i1" not in fs, fs # xpath in _get_inputs() - assert b"clickable2" not in fs, fs # xpath in _get_clickable() - - def test_from_response_submit_first_clickable(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response(response, formdata={"two": "2"}) - fs = _qs(req) - assert fs[b"clickable1"] == [b"clicked1"] - assert b"clickable2" not in fs, fs - assert fs[b"one"] == [b"1"] - assert fs[b"two"] == [b"2"] - - def test_from_response_submit_not_first_clickable(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response( - response, formdata={"two": "2"}, clickdata={"name": "clickable2"} - ) - fs = _qs(req) - assert fs[b"clickable2"] == [b"clicked2"] - assert b"clickable1" not in fs, fs - assert fs[b"one"] == [b"1"] - assert fs[b"two"] == [b"2"] - - def test_from_response_dont_submit_image_as_input(self): - response = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response(response, dont_click=True) - fs = _qs(req) - assert fs == {b"i1": [b"i1v"]} - - def test_from_response_dont_submit_reset_as_input(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response(response, dont_click=True) - fs = _qs(req) - assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} - - def test_from_response_clickdata_does_not_ignore_image(self): - response = _buildresponse( - """
- - -
""" - ) - req = self.request_class.from_response(response) - fs = _qs(req) - assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} - - def test_from_response_multiple_clickdata(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response( - response, clickdata={"name": "clickable", "value": "clicked2"} - ) - fs = _qs(req) - assert fs[b"clickable"] == [b"clicked2"] - assert fs[b"one"] == [b"clicked1"] - assert fs[b"two"] == [b"clicked2"] - - def test_from_response_unicode_clickdata(self): - response = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response( - response, clickdata={"name": "price in \u00a3"} - ) - fs = _qs(req, to_unicode=True) - assert fs["price in \u00a3"] - - def test_from_response_unicode_clickdata_latin1(self): - response = _buildresponse( - """
- - - - -
""", - encoding="latin1", - ) - req = self.request_class.from_response( - response, clickdata={"name": "price in \u00a5"} - ) - fs = _qs(req, to_unicode=True, encoding="latin1") - assert fs["price in \u00a5"] - - def test_from_response_multiple_forms_clickdata(self): - response = _buildresponse( - """
- - -
-
- - -
- """ - ) - req = self.request_class.from_response( - response, formname="form2", clickdata={"name": "clickable"} - ) - fs = _qs(req) - assert fs[b"clickable"] == [b"clicked2"] - assert fs[b"field2"] == [b"value2"] - assert b"field1" not in fs, fs - - def test_from_response_override_clickable(self): - response = _buildresponse( - """
""" - ) - req = self.request_class.from_response( - response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} - ) - fs = _qs(req) - assert fs[b"clickme"] == [b"two"] - - def test_from_response_dont_click(self): - response = _buildresponse( - """
- - - - -
""" - ) - r1 = self.request_class.from_response(response, dont_click=True) - fs = _qs(r1) - assert b"clickable1" not in fs, fs - assert b"clickable2" not in fs, fs - - def test_from_response_ambiguous_clickdata(self): - response = _buildresponse( - """ -
- - - - -
""" - ) - with pytest.raises( - ValueError, - match=r"Multiple elements found .* matching the criteria in clickdata", - ): - self.request_class.from_response(response, clickdata={"type": "submit"}) - - def test_from_response_non_matching_clickdata(self): - response = _buildresponse( - """
- -
""" - ) - with pytest.raises( - ValueError, match="No clickable element matching clickdata:" - ): - self.request_class.from_response( - response, clickdata={"nonexistent": "notme"} - ) - - def test_from_response_nr_index_clickdata(self): - response = _buildresponse( - """
- - -
- """ - ) - req = self.request_class.from_response(response, clickdata={"nr": 1}) - fs = _qs(req) - assert b"clickable2" in fs - assert b"clickable1" not in fs - - def test_from_response_invalid_nr_index_clickdata(self): - response = _buildresponse( - """
- -
- """ - ) - with pytest.raises( - ValueError, match="No clickable element matching clickdata:" - ): - self.request_class.from_response(response, clickdata={"nr": 1}) - - def test_from_response_errors_noform(self): - response = _buildresponse("""""") - with pytest.raises(ValueError, match="No
element found in"): - self.request_class.from_response(response) - - def test_from_response_invalid_html5(self): - response = _buildresponse( - """""" - """""" - """
""" - ) - req = self.request_class.from_response(response, formdata={"bar": "buz"}) - fs = _qs(req) - assert fs == {b"foo": [b"xxx"], b"bar": [b"buz"]} - - def test_from_response_errors_formnumber(self): - response = _buildresponse( - """
- - - -
""" - ) - with pytest.raises(IndexError): - self.request_class.from_response(response, formnumber=1) - - def test_from_response_noformname(self): - response = _buildresponse( - """
- - -
""" - ) - r1 = self.request_class.from_response(response, formdata={"two": "3"}) - assert r1.method == "POST" - assert r1.headers["Content-type"] == b"application/x-www-form-urlencoded" - fs = _qs(r1) - assert fs == {b"one": [b"1"], b"two": [b"3"]} - - def test_from_response_formname_exists(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response(response, formname="form2") - assert r1.method == "POST" - fs = _qs(r1) - assert fs == {b"four": [b"4"], b"three": [b"3"]} - - def test_from_response_formname_nonexistent(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - r1 = self.request_class.from_response(response, formname="form3") - assert r1.method == "POST" - fs = _qs(r1) - assert fs == {b"one": [b"1"]} - - def test_from_response_formname_errors_formnumber(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - with pytest.raises(IndexError): - self.request_class.from_response(response, formname="form3", formnumber=2) - - def test_from_response_formid_exists(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response(response, formid="form2") - assert r1.method == "POST" - fs = _qs(r1) - assert fs == {b"four": [b"4"], b"three": [b"3"]} - - def test_from_response_formname_nonexistent_fallback_formid(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response( - response, formname="form3", formid="form2" - ) - assert r1.method == "POST" - fs = _qs(r1) - assert fs == {b"four": [b"4"], b"three": [b"3"]} - - def test_from_response_formid_nonexistent(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - r1 = self.request_class.from_response(response, formid="form3") - assert r1.method == "POST" - fs = _qs(r1) - assert fs == {b"one": [b"1"]} - - def test_from_response_formid_errors_formnumber(self): - response = _buildresponse( - """
- -
-
- -
""" - ) - with pytest.raises(IndexError): - self.request_class.from_response(response, formid="form3", formnumber=2) - - def test_from_response_select(self): - res = _buildresponse( - """
- - - - - - - - - - - - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} - - def test_from_response_checkbox(self): - res = _buildresponse( - """
- - - - - - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} - - def test_from_response_input_text(self): - res = _buildresponse( - """
- - - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert fs == {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]} - - def test_from_response_input_hidden(self): - res = _buildresponse( - """
- - - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert fs == {b"i1": [b"i1v1"], b"i2": [b""]} - - def test_from_response_input_textarea(self): - res = _buildresponse( - """
- - - -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert fs == {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]} - - def test_from_response_descendants(self): - res = _buildresponse( - """
-
-
- - -
- - - - -
- -
""" - ) - req = self.request_class.from_response(res) - fs = _qs(req) - assert set(fs) == {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"} - - def test_from_response_xpath(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response( - response, formxpath="//form[@action='post.php']" - ) - fs = _qs(r1) - assert fs[b"one"] == [b"1"] - - r1 = self.request_class.from_response( - response, formxpath="//form/input[@name='four']" - ) - fs = _qs(r1) - assert fs[b"three"] == [b"3"] - - with pytest.raises(ValueError, match="No
element found with"): - self.request_class.from_response( - response, formxpath="//form/input[@name='abc']" - ) - - def test_from_response_unicode_xpath(self): - response = _buildresponse(b'
') - r = self.request_class.from_response( - response, formxpath="//form[@name='\u044a']" - ) - fs = _qs(r) - assert not fs - - xpath = "//form[@name='\u03b1']" - with pytest.raises(ValueError, match=re.escape(xpath)): - self.request_class.from_response(response, formxpath=xpath) - - def test_from_response_button_submit(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - assert req.method == "POST" - assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req) - assert fs[b"test1"] == [b"val1"] - assert fs[b"test2"] == [b"val2"] - assert fs[b"button1"] == [b"submit1"] - - def test_from_response_button_notype(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - assert req.method == "POST" - assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req) - assert fs[b"test1"] == [b"val1"] - assert fs[b"test2"] == [b"val2"] - assert fs[b"button1"] == [b"submit1"] - - def test_from_response_submit_novalue(self): - response = _buildresponse( - """
- - - Submit -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - assert req.method == "POST" - assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req) - assert fs[b"test1"] == [b"val1"] - assert fs[b"test2"] == [b"val2"] - assert fs[b"button1"] == [b""] - - def test_from_response_button_novalue(self): - response = _buildresponse( - """
- - - -
""", - url="http://www.example.com/this/list.html", - ) - req = self.request_class.from_response(response) - assert req.method == "POST" - assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" - assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req) - assert fs[b"test1"] == [b"val1"] - assert fs[b"test2"] == [b"val2"] - assert fs[b"button1"] == [b""] - - def test_html_base_form_action(self): - response = _buildresponse( - """ - - - - - -
-
- - - """, - url="http://a.com/", - ) - req = self.request_class.from_response(response) - assert req.url == "http://b.com/test_form" - - def test_spaces_in_action(self): - resp = _buildresponse('
') - req = self.request_class.from_response(resp) - assert req.url == "http://example.com/path" - - def test_from_response_css(self): - response = _buildresponse( - """
- - -
-
- - -
""" - ) - r1 = self.request_class.from_response( - response, formcss="form[action='post.php']" - ) - fs = _qs(r1) - assert fs[b"one"] == [b"1"] - - r1 = self.request_class.from_response(response, formcss="input[name='four']") - fs = _qs(r1) - assert fs[b"three"] == [b"3"] - - with pytest.raises(ValueError, match="No
element found with"): - self.request_class.from_response(response, formcss="input[name='abc']") - - def test_from_response_valid_form_methods(self): - form_methods = [ - [method, method] for method in self.request_class.valid_form_methods - ] - form_methods.append(["UNKNOWN", "GET"]) - - for method, expected in form_methods: - response = _buildresponse( - f'' - '' - "
" - ) - r = self.request_class.from_response(response) - assert r.method == expected - - def test_form_response_with_invalid_formdata_type_error(self): - """Test that a ValueError is raised for non-iterable and non-dict formdata input""" - response = _buildresponse( - """ -
- -
- """ - ) - with pytest.raises( - ValueError, match="formdata should be a dict or iterable of tuples" - ): - FormRequest.from_response(response, formdata=123) - - def test_form_response_with_custom_invalid_formdata_value_error(self): - """Test that a ValueError is raised for fault-inducing iterable formdata input""" - response = _buildresponse( - """ -
- -
- """ - ) - - with pytest.raises( - ValueError, match="formdata should be a dict or iterable of tuples" - ): - FormRequest.from_response(response, formdata=("a",)) - - def test_get_form_with_xpath_no_form_parent(self): - """Test that _get_from raised a ValueError when an XPath selects an element - not nested within a
and no parent is found""" - response = _buildresponse( - """ -
-

This paragraph is not inside a form.

-
- - -
- """ - ) - - with pytest.raises(ValueError, match="No
element found with"): - FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') - - -def _buildresponse(body, **kwargs): - kwargs.setdefault("body", body) - kwargs.setdefault("url", "http://example.com") - kwargs.setdefault("encoding", "utf-8") - return HtmlResponse(**kwargs) - - -def _qs(req, encoding="utf-8", to_unicode=False): - qs = req.body if req.method == "POST" else req.url.partition("?")[2] - uqs = unquote_to_bytes(qs) - if to_unicode: - uqs = uqs.decode(encoding) - return parse_qs(uqs, True) - - class TestXmlRpcRequest(TestRequest): request_class = XmlRpcRequest default_method = "POST" @@ -1539,184 +505,3 @@ class TestXmlRpcRequest(TestRequest): def test_latin1(self): self._test_request(params=("pas£",), encoding="latin1") - - -class TestJsonRequest(TestRequest): - request_class = JsonRequest - default_method = "GET" - default_headers = { - b"Content-Type": [b"application/json"], - b"Accept": [b"application/json, text/javascript, */*; q=0.01"], - } - - def test_data(self): - r1 = self.request_class(url="http://www.example.com/") - assert r1.body == b"" - - body = b"body" - r2 = self.request_class(url="http://www.example.com/", body=body) - assert r2.body == body - - data = { - "name": "value", - } - r3 = self.request_class(url="http://www.example.com/", data=data) - assert r3.body == to_bytes(json.dumps(data)) - - # empty data - r4 = self.request_class(url="http://www.example.com/", data=[]) - assert r4.body == to_bytes(json.dumps([])) - - def test_data_method(self): - # data is not passed - r1 = self.request_class(url="http://www.example.com/") - assert r1.method == "GET" - - body = b"body" - r2 = self.request_class(url="http://www.example.com/", body=body) - assert r2.method == "GET" - - data = { - "name": "value", - } - r3 = self.request_class(url="http://www.example.com/", data=data) - assert r3.method == "POST" - - # method passed explicitly - r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") - assert r4.method == "GET" - - r5 = self.request_class(url="http://www.example.com/", data=[]) - assert r5.method == "POST" - - def test_body_data(self): - """passing both body and data should result a warning""" - body = b"body" - data = { - "name": "value", - } - with warnings.catch_warnings(record=True) as _warnings: - r5 = self.request_class(url="http://www.example.com/", body=body, data=data) - assert r5.body == body - assert r5.method == "GET" - assert len(_warnings) == 1 - assert "data will be ignored" in str(_warnings[0].message) - - def test_empty_body_data(self): - """passing any body value and data should result a warning""" - data = { - "name": "value", - } - with warnings.catch_warnings(record=True) as _warnings: - r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) - assert r6.body == b"" - assert r6.method == "GET" - assert len(_warnings) == 1 - assert "data will be ignored" in str(_warnings[0].message) - - def test_body_none_data(self): - data = { - "name": "value", - } - with warnings.catch_warnings(record=True) as _warnings: - r7 = self.request_class(url="http://www.example.com/", body=None, data=data) - assert r7.body == to_bytes(json.dumps(data)) - assert r7.method == "POST" - assert len(_warnings) == 0 - - def test_body_data_none(self): - with warnings.catch_warnings(record=True) as _warnings: - r8 = self.request_class(url="http://www.example.com/", body=None, data=None) - assert r8.method == "GET" - assert len(_warnings) == 0 - - def test_dumps_sort_keys(self): - """Test that sort_keys=True is passed to json.dumps by default""" - data = { - "name": "value", - } - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - self.request_class(url="http://www.example.com/", data=data) - kwargs = mock_dumps.call_args[1] - assert kwargs["sort_keys"] is True - - def test_dumps_kwargs(self): - """Test that dumps_kwargs are passed to json.dumps""" - data = { - "name": "value", - } - dumps_kwargs = { - "ensure_ascii": True, - "allow_nan": True, - } - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - self.request_class( - url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs - ) - kwargs = mock_dumps.call_args[1] - assert kwargs["ensure_ascii"] is True - assert kwargs["allow_nan"] is True - - def test_replace_data(self): - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - r1 = self.request_class(url="http://www.example.com/", data=data1) - r2 = r1.replace(data=data2) - assert r2.body == to_bytes(json.dumps(data2)) - - def test_replace_sort_keys(self): - """Test that replace provides sort_keys=True to json.dumps""" - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - r1 = self.request_class(url="http://www.example.com/", data=data1) - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - r1.replace(data=data2) - kwargs = mock_dumps.call_args[1] - assert kwargs["sort_keys"] is True - - def test_replace_dumps_kwargs(self): - """Test that dumps_kwargs are provided to json.dumps when replace is called""" - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - dumps_kwargs = { - "ensure_ascii": True, - "allow_nan": True, - } - r1 = self.request_class( - url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs - ) - with mock.patch("json.dumps", return_value=b"") as mock_dumps: - r1.replace(data=data2) - kwargs = mock_dumps.call_args[1] - assert kwargs["ensure_ascii"] is True - assert kwargs["allow_nan"] is True - - def test_replacement_both_body_and_data_warns(self): - """Test that we get a warning if both body and data are passed""" - body1 = None - body2 = b"body" - data1 = { - "name1": "value1", - } - data2 = { - "name2": "value2", - } - r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) - - with warnings.catch_warnings(record=True) as _warnings: - r1.replace(data=data2, body=body2) - assert "Both body and data passed. data will be ignored" in str( - _warnings[0].message - ) diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py new file mode 100644 index 000000000..a4f87d50d --- /dev/null +++ b/tests/test_http_request_form.py @@ -0,0 +1,1028 @@ +from __future__ import annotations + +import re +from urllib.parse import parse_qs, unquote_to_bytes + +import pytest + +from scrapy.http import FormRequest, HtmlResponse +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_unicode +from tests.test_http_request import TestRequest + + +def _buildresponse(body, **kwargs): + kwargs.setdefault("body", body) + kwargs.setdefault("url", "http://example.com") + kwargs.setdefault("encoding", "utf-8") + return HtmlResponse(**kwargs) + + +def _qs(req, encoding="utf-8", to_unicode=False): + qs = req.body if req.method == "POST" else req.url.partition("?")[2] + uqs = unquote_to_bytes(qs) + if to_unicode: + uqs = uqs.decode(encoding) + return parse_qs(uqs, True) + + +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestFormRequest(TestRequest): + request_class = FormRequest # type: ignore[assignment] + + def assertQueryEqual(self, first, second, msg=None): + first = to_unicode(first).split("&") + second = to_unicode(second).split("&") + assert sorted(first) == sorted(second), msg + + def test_empty_formdata(self): + r1 = self.request_class("http://www.example.com", formdata={}) + assert r1.body == b"" + + def test_formdata_overrides_querystring(self): + data = (("a", "one"), ("a", "two"), ("b", "2")) + url = self.request_class( + "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data + ).url.split("#", maxsplit=1)[0] + fs = _qs(self.request_class(url, method="GET", formdata=data)) + assert set(fs[b"a"]) == {b"one", b"two"} + assert fs[b"b"] == [b"2"] + assert fs.get(b"c") is None + + data = {"a": "1", "b": "2"} + fs = _qs( + self.request_class("http://www.example.com/", method="GET", formdata=data) + ) + assert fs[b"a"] == [b"1"] + assert fs[b"b"] == [b"2"] + + def test_default_encoding_bytes(self): + # using default encoding (utf-8) + data = {b"one": b"two", b"price": b"\xc2\xa3 100"} + r2 = self.request_class("http://www.example.com", formdata=data) + assert r2.method == "POST" + assert r2.encoding == "utf-8" + self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_default_encoding_textual_data(self): + # using default encoding (utf-8) + data = {"µ one": "two", "price": "£ 100"} + r2 = self.request_class("http://www.example.com", formdata=data) + assert r2.method == "POST" + assert r2.encoding == "utf-8" + self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_default_encoding_mixed_data(self): + # using default encoding (utf-8) + data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} + r2 = self.request_class("http://www.example.com", formdata=data) + assert r2.method == "POST" + assert r2.encoding == "utf-8" + self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_custom_encoding_bytes(self): + data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} + r2 = self.request_class( + "http://www.example.com", formdata=data, encoding="latin1" + ) + assert r2.method == "POST" + assert r2.encoding == "latin1" + self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") + assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" + + def test_custom_encoding_textual_data(self): + data = {"price": "£ 100"} + r3 = self.request_class( + "http://www.example.com", formdata=data, encoding="latin1" + ) + assert r3.encoding == "latin1" + assert r3.body == b"price=%A3+100" + + def test_multi_key_values(self): + # using multiples values for a single key + data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} + r3 = self.request_class("http://www.example.com", formdata=data) + self.assertQueryEqual( + r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" + ) + + def test_from_response_post(self): + response = _buildresponse( + b""" + + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] + + def test_from_response_post_nonascii_bytes_utf8(self): + response = _buildresponse( + b"""
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req, to_unicode=True) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] + + def test_from_response_post_nonascii_bytes_latin1(self): + response = _buildresponse( + b"""
+ + + +
""", + url="http://www.example.com/this/list.html", + encoding="latin1", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req, to_unicode=True, encoding="latin1") + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] + + def test_from_response_post_nonascii_unicode(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + + assert req.method == "POST" + assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req, to_unicode=True) + assert set(fs["test £"]) == {"val1", "val2"} + assert set(fs["one"]) == {"two", "three"} + assert fs["test2"] == ["xxx µ"] + assert fs["six"] == ["seven"] + + def test_from_response_duplicate_form_key(self): + response = _buildresponse("
", url="http://www.example.com") + req = self.request_class.from_response( + response=response, + method="GET", + formdata=(("foo", "bar"), ("foo", "baz")), + ) + assert urlparse_cached(req).hostname == "www.example.com" + assert urlparse_cached(req).query == "foo=bar&foo=baz" + + def test_from_response_override_duplicate_form_key(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response( + response, formdata=(("two", "2"), ("two", "4")) + ) + fs = _qs(req) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2", b"4"] + + def test_from_response_extra_headers(self): + response = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response( + response=response, + formdata={"one": ["two", "three"], "six": "seven"}, + headers={"Accept-Encoding": "gzip,deflate"}, + ) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.headers["Accept-Encoding"] == b"gzip,deflate" + + def test_from_response_get(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + r1 = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + assert r1.method == "GET" + assert urlparse_cached(r1).hostname == "www.example.com" + assert urlparse_cached(r1).path == "/this/get.php" + fs = _qs(r1) + assert set(fs[b"test"]) == {b"val1", b"val2"} + assert set(fs[b"one"]) == {b"two", b"three"} + assert fs[b"test2"] == [b"xxx"] + assert fs[b"six"] == [b"seven"] + + def test_from_response_override_params(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response(response, formdata={"two": "2"}) + fs = _qs(req) + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] + + def test_from_response_drop_params(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response(response, formdata={"two": None}) + fs = _qs(req) + assert fs[b"one"] == [b"1"] + assert b"two" not in fs + + def test_from_response_override_method(self): + response = _buildresponse( + """ +
+ """ + ) + request = FormRequest.from_response(response) + assert request.method == "GET" + request = FormRequest.from_response(response, method="POST") + assert request.method == "POST" + + def test_from_response_override_url(self): + response = _buildresponse( + """ +
+ """ + ) + request = FormRequest.from_response(response) + assert request.url == "http://example.com/app" + request = FormRequest.from_response(response, url="http://foo.bar/absolute") + assert request.url == "http://foo.bar/absolute" + request = FormRequest.from_response(response, url="/relative") + assert request.url == "http://example.com/relative" + + def test_from_response_case_insensitive(self): + response = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response(response) + fs = _qs(req) + assert fs[b"clickable1"] == [b"clicked1"] + assert b"i1" not in fs, fs # xpath in _get_inputs() + assert b"clickable2" not in fs, fs # xpath in _get_clickable() + + def test_from_response_submit_first_clickable(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response(response, formdata={"two": "2"}) + fs = _qs(req) + assert fs[b"clickable1"] == [b"clicked1"] + assert b"clickable2" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] + + def test_from_response_submit_not_first_clickable(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response( + response, formdata={"two": "2"}, clickdata={"name": "clickable2"} + ) + fs = _qs(req) + assert fs[b"clickable2"] == [b"clicked2"] + assert b"clickable1" not in fs, fs + assert fs[b"one"] == [b"1"] + assert fs[b"two"] == [b"2"] + + def test_from_response_dont_submit_image_as_input(self): + response = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response(response, dont_click=True) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"]} + + def test_from_response_dont_submit_reset_as_input(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response(response, dont_click=True) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""" + ) + req = self.request_class.from_response(response) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"], b"i2": [b"i2v"]} + + def test_from_response_multiple_clickdata(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response( + response, clickdata={"name": "clickable", "value": "clicked2"} + ) + fs = _qs(req) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"one"] == [b"clicked1"] + assert fs[b"two"] == [b"clicked2"] + + def test_from_response_unicode_clickdata(self): + response = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response( + response, clickdata={"name": "price in \u00a3"} + ) + fs = _qs(req, to_unicode=True) + assert fs["price in \u00a3"] + + def test_from_response_unicode_clickdata_latin1(self): + response = _buildresponse( + """
+ + + + +
""", + encoding="latin1", + ) + req = self.request_class.from_response( + response, clickdata={"name": "price in \u00a5"} + ) + fs = _qs(req, to_unicode=True, encoding="latin1") + assert fs["price in \u00a5"] + + def test_from_response_multiple_forms_clickdata(self): + response = _buildresponse( + """
+ + +
+
+ + +
+ """ + ) + req = self.request_class.from_response( + response, formname="form2", clickdata={"name": "clickable"} + ) + fs = _qs(req) + assert fs[b"clickable"] == [b"clicked2"] + assert fs[b"field2"] == [b"value2"] + assert b"field1" not in fs, fs + + def test_from_response_override_clickable(self): + response = _buildresponse( + """
""" + ) + req = self.request_class.from_response( + response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} + ) + fs = _qs(req) + assert fs[b"clickme"] == [b"two"] + + def test_from_response_dont_click(self): + response = _buildresponse( + """
+ + + + +
""" + ) + r1 = self.request_class.from_response(response, dont_click=True) + fs = _qs(r1) + assert b"clickable1" not in fs, fs + assert b"clickable2" not in fs, fs + + def test_from_response_ambiguous_clickdata(self): + response = _buildresponse( + """ +
+ + + + +
""" + ) + with pytest.raises( + ValueError, + match=r"Multiple elements found .* matching the criteria in clickdata", + ): + self.request_class.from_response(response, clickdata={"type": "submit"}) + + def test_from_response_non_matching_clickdata(self): + response = _buildresponse( + """
+ +
""" + ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response( + response, clickdata={"nonexistent": "notme"} + ) + + def test_from_response_nr_index_clickdata(self): + response = _buildresponse( + """
+ + +
+ """ + ) + req = self.request_class.from_response(response, clickdata={"nr": 1}) + fs = _qs(req) + assert b"clickable2" in fs + assert b"clickable1" not in fs + + def test_from_response_invalid_nr_index_clickdata(self): + response = _buildresponse( + """
+ +
+ """ + ) + with pytest.raises( + ValueError, match="No clickable element matching clickdata:" + ): + self.request_class.from_response(response, clickdata={"nr": 1}) + + def test_from_response_errors_noform(self): + response = _buildresponse("""""") + with pytest.raises(ValueError, match="No
element found in"): + self.request_class.from_response(response) + + def test_from_response_invalid_html5(self): + response = _buildresponse( + """""" + """""" + """
""" + ) + req = self.request_class.from_response(response, formdata={"bar": "buz"}) + fs = _qs(req) + assert fs == {b"foo": [b"xxx"], b"bar": [b"buz"]} + + def test_from_response_errors_formnumber(self): + response = _buildresponse( + """
+ + + +
""" + ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formnumber=1) + + def test_from_response_noformname(self): + response = _buildresponse( + """
+ + +
""" + ) + r1 = self.request_class.from_response(response, formdata={"two": "3"}) + assert r1.method == "POST" + assert r1.headers["Content-type"] == b"application/x-www-form-urlencoded" + fs = _qs(r1) + assert fs == {b"one": [b"1"], b"two": [b"3"]} + + def test_from_response_formname_exists(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response(response, formname="form2") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"four": [b"4"], b"three": [b"3"]} + + def test_from_response_formname_nonexistent(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + r1 = self.request_class.from_response(response, formname="form3") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"one": [b"1"]} + + def test_from_response_formname_errors_formnumber(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formname="form3", formnumber=2) + + def test_from_response_formid_exists(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response(response, formid="form2") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"four": [b"4"], b"three": [b"3"]} + + def test_from_response_formname_nonexistent_fallback_formid(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response( + response, formname="form3", formid="form2" + ) + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"four": [b"4"], b"three": [b"3"]} + + def test_from_response_formid_nonexistent(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + r1 = self.request_class.from_response(response, formid="form3") + assert r1.method == "POST" + fs = _qs(r1) + assert fs == {b"one": [b"1"]} + + def test_from_response_formid_errors_formnumber(self): + response = _buildresponse( + """
+ +
+
+ +
""" + ) + with pytest.raises(IndexError): + self.request_class.from_response(response, formid="form3", formnumber=2) + + def test_from_response_select(self): + res = _buildresponse( + """
+ + + + + + + + + + + + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} + + def test_from_response_checkbox(self): + res = _buildresponse( + """
+ + + + + + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"iv2"], b"i2": [b"on"]} + + def test_from_response_input_text(self): + res = _buildresponse( + """
+ + + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]} + + def test_from_response_input_hidden(self): + res = _buildresponse( + """
+ + + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"i1v1"], b"i2": [b""]} + + def test_from_response_input_textarea(self): + res = _buildresponse( + """
+ + + +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert fs == {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]} + + def test_from_response_descendants(self): + res = _buildresponse( + """
+
+
+ + +
+ + + + +
+ +
""" + ) + req = self.request_class.from_response(res) + fs = _qs(req) + assert set(fs) == {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"} + + def test_from_response_xpath(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response( + response, formxpath="//form[@action='post.php']" + ) + fs = _qs(r1) + assert fs[b"one"] == [b"1"] + + r1 = self.request_class.from_response( + response, formxpath="//form/input[@name='four']" + ) + fs = _qs(r1) + assert fs[b"three"] == [b"3"] + + with pytest.raises(ValueError, match="No
element found with"): + self.request_class.from_response( + response, formxpath="//form/input[@name='abc']" + ) + + def test_from_response_unicode_xpath(self): + response = _buildresponse(b'
') + r = self.request_class.from_response( + response, formxpath="//form[@name='\u044a']" + ) + fs = _qs(r) + assert not fs + + xpath = "//form[@name='\u03b1']" + with pytest.raises(ValueError, match=re.escape(xpath)): + self.request_class.from_response(response, formxpath=xpath) + + def test_from_response_button_submit(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] + + def test_from_response_button_notype(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b"submit1"] + + def test_from_response_submit_novalue(self): + response = _buildresponse( + """
+ + + Submit +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] + + def test_from_response_button_novalue(self): + response = _buildresponse( + """
+ + + +
""", + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response(response) + assert req.method == "POST" + assert req.headers["Content-type"] == b"application/x-www-form-urlencoded" + assert req.url == "http://www.example.com/this/post.php" + fs = _qs(req) + assert fs[b"test1"] == [b"val1"] + assert fs[b"test2"] == [b"val2"] + assert fs[b"button1"] == [b""] + + def test_html_base_form_action(self): + response = _buildresponse( + """ + + + + + +
+
+ + + """, + url="http://a.com/", + ) + req = self.request_class.from_response(response) + assert req.url == "http://b.com/test_form" + + def test_spaces_in_action(self): + resp = _buildresponse('
') + req = self.request_class.from_response(resp) + assert req.url == "http://example.com/path" + + def test_from_response_css(self): + response = _buildresponse( + """
+ + +
+
+ + +
""" + ) + r1 = self.request_class.from_response( + response, formcss="form[action='post.php']" + ) + fs = _qs(r1) + assert fs[b"one"] == [b"1"] + + r1 = self.request_class.from_response(response, formcss="input[name='four']") + fs = _qs(r1) + assert fs[b"three"] == [b"3"] + + with pytest.raises(ValueError, match="No
element found with"): + self.request_class.from_response(response, formcss="input[name='abc']") + + def test_from_response_valid_form_methods(self): + form_methods = [ + [method, method] for method in self.request_class.valid_form_methods + ] + form_methods.append(["UNKNOWN", "GET"]) + + for method, expected in form_methods: + response = _buildresponse( + f'' + '' + "
" + ) + r = self.request_class.from_response(response) + assert r.method == expected + + def test_form_response_with_invalid_formdata_type_error(self): + """Test that a ValueError is raised for non-iterable and non-dict formdata input""" + response = _buildresponse( + """ +
+ +
+ """ + ) + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): + FormRequest.from_response(response, formdata=123) + + def test_form_response_with_custom_invalid_formdata_value_error(self): + """Test that a ValueError is raised for fault-inducing iterable formdata input""" + response = _buildresponse( + """ +
+ +
+ """ + ) + + with pytest.raises( + ValueError, match="formdata should be a dict or iterable of tuples" + ): + FormRequest.from_response(response, formdata=("a",)) + + def test_get_form_with_xpath_no_form_parent(self): + """Test that _get_from raised a ValueError when an XPath selects an element + not nested within a
and no parent is found""" + response = _buildresponse( + """ +
+

This paragraph is not inside a form.

+
+ + +
+ """ + ) + + with pytest.raises(ValueError, match="No
element found with"): + FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') diff --git a/tests/test_http_request_json.py b/tests/test_http_request_json.py new file mode 100644 index 000000000..65022afe1 --- /dev/null +++ b/tests/test_http_request_json.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import json +import warnings +from unittest import mock + +import pytest + +from scrapy.http import JsonRequest +from scrapy.utils.python import to_bytes +from tests.test_http_request import TestRequest + + +class TestJsonRequest(TestRequest): + request_class = JsonRequest + default_method = "GET" + default_headers = { + b"Content-Type": [b"application/json"], + b"Accept": [b"application/json, text/javascript, */*; q=0.01"], + } + + def test_data(self): + r1 = self.request_class(url="http://www.example.com/") + assert r1.body == b"" + + body = b"body" + r2 = self.request_class(url="http://www.example.com/", body=body) + assert r2.body == body + + data = { + "name": "value", + } + r3 = self.request_class(url="http://www.example.com/", data=data) + assert r3.body == to_bytes(json.dumps(data)) + + # empty data + r4 = self.request_class(url="http://www.example.com/", data=[]) + assert r4.body == to_bytes(json.dumps([])) + + def test_data_method(self): + # data is not passed + r1 = self.request_class(url="http://www.example.com/") + assert r1.method == "GET" + + body = b"body" + r2 = self.request_class(url="http://www.example.com/", body=body) + assert r2.method == "GET" + + data = { + "name": "value", + } + r3 = self.request_class(url="http://www.example.com/", data=data) + assert r3.method == "POST" + + # method passed explicitly + r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") + assert r4.method == "GET" + + r5 = self.request_class(url="http://www.example.com/", data=[]) + assert r5.method == "POST" + + def test_body_data(self): + """passing both body and data should result a warning""" + body = b"body" + data = { + "name": "value", + } + with pytest.warns(UserWarning, match="data will be ignored"): + r5 = self.request_class(url="http://www.example.com/", body=body, data=data) + assert r5.body == body + assert r5.method == "GET" + + def test_empty_body_data(self): + """passing any body value and data should result a warning""" + data = { + "name": "value", + } + with pytest.warns(UserWarning, match="data will be ignored"): + r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) + assert r6.body == b"" + assert r6.method == "GET" + + def test_body_none_data(self): + data = { + "name": "value", + } + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", category=UserWarning, message="Both body and data passed" + ) + r7 = self.request_class(url="http://www.example.com/", body=None, data=data) + assert r7.body == to_bytes(json.dumps(data)) + assert r7.method == "POST" + + def test_body_data_none(self): + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", category=UserWarning, message="Both body and data passed" + ) + r8 = self.request_class(url="http://www.example.com/", body=None, data=None) + assert r8.method == "GET" + + def test_dumps_sort_keys(self): + """Test that sort_keys=True is passed to json.dumps by default""" + data = { + "name": "value", + } + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + self.request_class(url="http://www.example.com/", data=data) + kwargs = mock_dumps.call_args[1] + assert kwargs["sort_keys"] is True + + def test_dumps_kwargs(self): + """Test that dumps_kwargs are passed to json.dumps""" + data = { + "name": "value", + } + dumps_kwargs = { + "ensure_ascii": True, + "allow_nan": True, + } + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + self.request_class( + url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs + ) + kwargs = mock_dumps.call_args[1] + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True + + def test_replace_data(self): + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + r2 = r1.replace(data=data2) + assert r2.body == to_bytes(json.dumps(data2)) + + def test_replace_sort_keys(self): + """Test that replace provides sort_keys=True to json.dumps""" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1) + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + assert kwargs["sort_keys"] is True + + def test_replace_dumps_kwargs(self): + """Test that dumps_kwargs are provided to json.dumps when replace is called""" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + dumps_kwargs = { + "ensure_ascii": True, + "allow_nan": True, + } + r1 = self.request_class( + url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs + ) + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + r1.replace(data=data2) + kwargs = mock_dumps.call_args[1] + assert kwargs["ensure_ascii"] is True + assert kwargs["allow_nan"] is True + + def test_replacement_both_body_and_data_warns(self): + """Test that we get a warning if both body and data are passed""" + body1 = None + body2 = b"body" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) + + with pytest.warns(UserWarning, match="data will be ignored"): + r1.replace(data=data2, body=body2) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 8acdf25ec..079c547c7 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,27 +1,21 @@ -import codecs -from unittest import mock +from __future__ import annotations + +from typing import TYPE_CHECKING import pytest -from packaging.version import Version as parse_version -from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding from scrapy.exceptions import NotSupported -from scrapy.http import ( - Headers, - HtmlResponse, - Request, - Response, - TextResponse, - XmlResponse, -) +from scrapy.http import Headers, Request, Response from scrapy.link import Link -from scrapy.selector import Selector -from scrapy.utils.python import to_unicode +from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS from tests import get_testdata +if TYPE_CHECKING: + from collections.abc import Iterable -class TestResponseBase: + +class TestResponse: response_class = Response def test_init(self): @@ -172,6 +166,53 @@ class TestResponseBase: with pytest.raises(AttributeError): r.body = "xxx" + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + response = self.response_class("http://example.com") + + response.request = Request("http://example.com") + + assert response._flags is None + assert response.flags == [] + assert response.flags is response.flags + assert response._flags == [] + original_flags = response.flags + response.flags = None + assert response._flags is None + assert response.flags == [] + assert response.flags is not original_flags + + assert response._headers is None + assert response.headers == {} + assert response.headers is response.headers + assert isinstance(response.headers, Headers) + assert isinstance(response._headers, Headers) + original_headers = response.headers + response.headers = None + assert response._headers is None + assert response.headers == {} + assert response._headers == {} + assert response.headers is not original_headers + + def test_setters(self): + response = self.response_class("http://example.com") + + response.flags = ["f1"] + assert response.flags == ["f1"] + + headers = Headers({b"X-Test": b"1"}) + response.headers = headers + assert response._headers is headers + response.headers = {b"A": b"b"} + assert isinstance(response.headers, Headers) + assert response._headers[b"A"] == b"b" + def test_urljoin(self): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" joined = self.response_class("http://www.example.com").urljoin("/test") @@ -214,7 +255,7 @@ class TestResponseBase: r.follow(None) @pytest.mark.xfail( - parse_version(w3lib_version) < parse_version("2.1.1"), + not W3LIB_STRIPS_URLS, reason="https://github.com/scrapy/w3lib/pull/207", strict=True, ) @@ -222,7 +263,7 @@ class TestResponseBase: self._assert_followed_url("foo ", "http://example.com/foo") @pytest.mark.xfail( - parse_version(w3lib_version) < parse_version("2.1.1"), + not W3LIB_STRIPS_URLS, reason="https://github.com/scrapy/w3lib/pull/207", strict=True, ) @@ -290,16 +331,26 @@ class TestResponseBase: with pytest.raises(ValueError, match="url can't be None"): list(r.follow_all(urls=[None])) + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_all_whitespace(self): relative = ["foo ", "bar ", "foo/bar ", "bar/foo "] absolute = [ - "http://example.com/foo%20", - "http://example.com/bar%20", - "http://example.com/foo/bar%20", - "http://example.com/bar/foo%20", + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", ] self._assert_followed_all_urls(relative, absolute) + @pytest.mark.xfail( + not W3LIB_STRIPS_URLS, + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_all_whitespace_links(self): absolute = [ "http://example.com/foo ", @@ -307,8 +358,8 @@ class TestResponseBase: "http://example.com/foo/bar ", "http://example.com/bar/foo ", ] - links = map(Link, absolute) - expected = [u.replace(" ", "%20") for u in absolute] + links = [Link(u) for u in absolute] + expected = [u.strip() for u in absolute] self._assert_followed_all_urls(links, expected) def test_follow_all_flags(self): @@ -322,698 +373,36 @@ class TestResponseBase: for req in fol: assert req.flags == ["cached", "allowed"] - def _assert_followed_url(self, follow_obj, target_url, response=None): + def _assert_followed_url( + self, + follow_obj: str | Link, + target_url: str, + response: Response | None = None, + encoding: str | None = None, + ) -> None: if response is None: response = self._links_response() req = response.follow(follow_obj) assert req.url == target_url - return req + if encoding is not None: + assert req.encoding == encoding - def _assert_followed_all_urls(self, follow_obj, target_urls, response=None): + def _assert_followed_all_urls( + self, + follow_obj: Iterable[str | Link], + target_urls: Iterable[str], + response: Response | None = None, + ) -> None: if response is None: response = self._links_response() followed = response.follow_all(follow_obj) - for req, target in zip(followed, target_urls, strict=False): + for req, target in zip(followed, target_urls, strict=True): assert req.url == target - yield req - def _links_response(self): + def _links_response(self) -> Response: body = get_testdata("link_extractor", "linkextractor.html") return self.response_class("http://example.com/index", body=body) - def _links_response_no_href(self): + def _links_response_no_href(self) -> Response: body = get_testdata("link_extractor", "linkextractor_no_href.html") return self.response_class("http://example.com/index", body=body) - - -class TestTextResponse(TestResponseBase): - response_class = TextResponse - - def test_replace(self): - super().test_replace() - r1 = self.response_class( - "http://www.example.com", body="hello", encoding="cp852" - ) - r2 = r1.replace(url="http://www.example.com/other") - r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") - - assert isinstance(r2, self.response_class) - assert r2.url == "http://www.example.com/other" - self._assert_response_encoding(r2, "cp852") - assert r3.url == "http://www.example.com/other" - assert r3._declared_encoding() == "latin1" - - def test_unicode_url(self): - # instantiate with unicode url without encoding (should set default encoding) - resp = self.response_class("http://www.example.com/") - self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) - - # make sure urls are converted to str - resp = self.response_class(url="http://www.example.com/", encoding="utf-8") - assert isinstance(resp.url, str) - - resp = self.response_class( - url="http://www.example.com/price/\xa3", encoding="utf-8" - ) - assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") - resp = self.response_class( - url="http://www.example.com/price/\xa3", encoding="latin-1" - ) - assert resp.url == "http://www.example.com/price/\xa3" - resp = self.response_class( - "http://www.example.com/price/\xa3", - headers={"Content-type": ["text/html; charset=utf-8"]}, - ) - assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") - resp = self.response_class( - "http://www.example.com/price/\xa3", - headers={"Content-type": ["text/html; charset=iso-8859-1"]}, - ) - assert resp.url == "http://www.example.com/price/\xa3" - - def test_unicode_body(self): - unicode_string = ( - "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " - "\u0442\u0435\u043a\u0441\u0442" - ) - with pytest.raises(TypeError): - self.response_class("http://www.example.com", body="unicode body") - - original_string = unicode_string.encode("cp1251") - r1 = self.response_class( - "http://www.example.com", body=original_string, encoding="cp1251" - ) - - # check response.text - assert isinstance(r1.text, str) - assert r1.text == unicode_string - - def test_encoding(self): - r1 = self.response_class( - "http://www.example.com", - body=b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=utf-8"]}, - ) - r2 = self.response_class( - "http://www.example.com", encoding="utf-8", body="\xa3" - ) - r3 = self.response_class( - "http://www.example.com", - body=b"\xa3", - headers={"Content-type": ["text/html; charset=iso-8859-1"]}, - ) - r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") - r5 = self.response_class( - "http://www.example.com", - body=b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=None"]}, - ) - r6 = self.response_class( - "http://www.example.com", - body=b"\xa8D", - headers={"Content-type": ["text/html; charset=gb2312"]}, - ) - r7 = self.response_class( - "http://www.example.com", - body=b"\xa8D", - headers={"Content-type": ["text/html; charset=gbk"]}, - ) - r8 = self.response_class( - "http://www.example.com", - body=codecs.BOM_UTF8 + b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=cp1251"]}, - ) - r9 = self.response_class( - "http://www.example.com", - body=b"\x80", - headers={ - "Content-type": [b"application/x-download; filename=\x80dummy.txt"] - }, - ) - - assert r1._headers_encoding() == "utf-8" - assert r2._headers_encoding() is None - assert r2._declared_encoding() == "utf-8" - self._assert_response_encoding(r2, "utf-8") - assert r3._headers_encoding() == "cp1252" - assert r3._declared_encoding() == "cp1252" - assert r4._headers_encoding() is None - assert r5._headers_encoding() is None - assert r8._headers_encoding() == "cp1251" - assert r9._headers_encoding() is None - assert r8._declared_encoding() == "utf-8" - assert r9._declared_encoding() is None - self._assert_response_encoding(r5, "utf-8") - self._assert_response_encoding(r8, "utf-8") - self._assert_response_encoding(r9, "cp1252") - assert r4._body_inferred_encoding() is not None - assert r4._body_inferred_encoding() != "ascii" - self._assert_response_values(r1, "utf-8", "\xa3") - self._assert_response_values(r2, "utf-8", "\xa3") - self._assert_response_values(r3, "iso-8859-1", "\xa3") - self._assert_response_values(r6, "gb18030", "\u2015") - self._assert_response_values(r7, "gb18030", "\u2015") - self._assert_response_values(r9, "cp1252", "€") - - # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - with pytest.raises(TypeError): - self.response_class("http://www.example.com", body="\xa3") - - def test_declared_encoding_invalid(self): - """Check that unknown declared encodings are ignored""" - r = self.response_class( - "http://www.example.com", - headers={"Content-type": ["text/html; charset=UNKNOWN"]}, - body=b"\xc2\xa3", - ) - assert r._declared_encoding() is None - self._assert_response_values(r, "utf-8", "\xa3") - - def test_utf16(self): - """Test utf-16 because UnicodeDammit is known to have problems with""" - r = self.response_class( - "http://www.example.com", - body=b"\xff\xfeh\x00i\x00", - encoding="utf-16", - ) - self._assert_response_values(r, "utf-16", "hi") - - def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): - r6 = self.response_class( - "http://www.example.com", - headers={"Content-type": ["text/html; charset=utf-8"]}, - body=b"\xef\xbb\xbfWORD\xe3\xab", - ) - assert r6.encoding == "utf-8" - assert r6.text in { - "WORD\ufffd\ufffd", # w3lib < 1.19.0 - "WORD\ufffd", # w3lib >= 1.19.0 - } - - def test_bom_is_removed_from_body(self): - # Inferring encoding from body also cache decoded body as sideeffect, - # this test tries to ensure that calling response.encoding and - # response.text in indistinct order doesn't affect final - # response.text in indistinct order doesn't affect final - # values for encoding and decoded body. - url = "http://example.com" - body = b"\xef\xbb\xbfWORD" - headers = {"Content-type": ["text/html; charset=utf-8"]} - - # Test response without content-type and BOM encoding - response = self.response_class(url, body=body) - assert response.encoding == "utf-8" - assert response.text == "WORD" - response = self.response_class(url, body=body) - assert response.text == "WORD" - assert response.encoding == "utf-8" - - # Body caching sideeffect isn't triggered when encoding is declared in - # content-type header but BOM still need to be removed from decoded - # body - response = self.response_class(url, headers=headers, body=body) - assert response.encoding == "utf-8" - assert response.text == "WORD" - response = self.response_class(url, headers=headers, body=body) - assert response.text == "WORD" - assert response.encoding == "utf-8" - - def test_replace_wrong_encoding(self): - """Test invalid chars are replaced properly""" - r = self.response_class( - "http://www.example.com", - encoding="utf-8", - body=b"PREFIX\xe3\xabSUFFIX", - ) - # XXX: Policy for replacing invalid chars may suffer minor variations - # but it should always contain the unicode replacement char ('\ufffd') - assert "\ufffd" in r.text, repr(r.text) - assert "PREFIX" in r.text, repr(r.text) - assert "SUFFIX" in r.text, repr(r.text) - - # Do not destroy html tags due to encoding bugs - r = self.response_class( - "http://example.com", - encoding="utf-8", - body=b"\xf0value", - ) - assert "value" in r.text, repr(r.text) - - # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse - # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') - # assert '\ufffd' in r.text, repr(r.text) - - def test_selector(self): - body = b"Some page" - response = self.response_class("http://www.example.com", body=body) - - assert isinstance(response.selector, Selector) - assert response.selector.type == "html" - assert response.selector is response.selector # property is cached - assert response.selector.response is response - - assert response.selector.xpath("//title/text()").getall() == ["Some page"] - assert response.selector.css("title::text").getall() == ["Some page"] - assert response.selector.re("Some (.*)") == ["page"] - - def test_selector_shortcuts(self): - body = b"Some page" - response = self.response_class("http://www.example.com", body=body) - - assert ( - response.xpath("//title/text()").getall() - == response.selector.xpath("//title/text()").getall() - ) - assert ( - response.css("title::text").getall() - == response.selector.css("title::text").getall() - ) - - def test_selector_shortcuts_kwargs(self): - body = b'Some page

A nice paragraph.

' - response = self.response_class("http://www.example.com", body=body) - - assert ( - response.xpath( - "normalize-space(//p[@class=$pclass])", pclass="content" - ).getall() - == response.xpath('normalize-space(//p[@class="content"])').getall() - ) - assert ( - response.xpath( - "//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", - pcount=1, - ).getall() - == response.xpath( - '//title[count(following::p[@class="content"])=1]/text()' - ).getall() - ) - - def test_urljoin_with_base_url(self): - """Test urljoin shortcut which also evaluates base-url through get_base_url().""" - body = b'' - joined = self.response_class("http://www.example.com", body=body).urljoin( - "/test" - ) - absolute = "https://example.net/test" - assert joined == absolute - - body = b'' - joined = self.response_class("http://www.example.com", body=body).urljoin( - "test" - ) - absolute = "http://www.example.com/test" - assert joined == absolute - - body = b'' - joined = self.response_class("http://www.example.com", body=body).urljoin( - "test" - ) - absolute = "http://www.example.com/elsewhere/test" - assert joined == absolute - - def test_follow_selector(self): - resp = self._links_response() - urls = [ - "http://example.com/sample2.html", - "http://example.com/sample3.html", - "http://example.com/sample3.html", - "http://example.com/sample3.html", - "http://example.com/sample3.html#foo", - "http://www.google.com/something", - "http://example.com/innertag.html", - ] - - # select elements - for sellist in [resp.css("a"), resp.xpath("//a")]: - for sel, url in zip(sellist, urls, strict=False): - self._assert_followed_url(sel, url, response=resp) - - # select elements - self._assert_followed_url( - Selector(text='').css("link")[0], - "http://example.com/foo", - response=resp, - ) - - # href attributes should work - for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]: - for sel, url in zip(sellist, urls, strict=False): - self._assert_followed_url(sel, url, response=resp) - - # non-a elements are not supported - with pytest.raises( - ValueError, match="Only and elements are supported" - ): - resp.follow(resp.css("div")[0]) - - def test_follow_selector_list(self): - resp = self._links_response() - with pytest.raises(ValueError, match="SelectorList"): - resp.follow(resp.css("a")) - - def test_follow_selector_invalid(self): - resp = self._links_response() - with pytest.raises(ValueError, match="Unsupported"): - resp.follow(resp.xpath("count(//div)")[0]) - - def test_follow_selector_attribute(self): - resp = self._links_response() - for src in resp.css("img::attr(src)"): - self._assert_followed_url(src, "http://example.com/sample2.jpg") - - def test_follow_selector_no_href(self): - resp = self.response_class( - url="http://example.com", - body=b"click me", - ) - with pytest.raises(ValueError, match="no href"): - resp.follow(resp.css("a")[0]) - - def test_follow_whitespace_selector(self): - resp = self.response_class( - "http://example.com", - body=b"""click me""", - ) - self._assert_followed_url( - resp.css("a")[0], "http://example.com/foo", response=resp - ) - self._assert_followed_url( - resp.css("a::attr(href)")[0], - "http://example.com/foo", - response=resp, - ) - - def test_follow_encoding(self): - resp1 = self.response_class( - "http://example.com", - encoding="utf8", - body='click me'.encode(), - ) - req = self._assert_followed_url( - resp1.css("a")[0], - "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", - response=resp1, - ) - assert req.encoding == "utf8" - - resp2 = self.response_class( - "http://example.com", - encoding="cp1251", - body='click me'.encode( - "cp1251" - ), - ) - req = self._assert_followed_url( - resp2.css("a")[0], - "http://example.com/foo?%EF%F0%E8%E2%E5%F2", - response=resp2, - ) - assert req.encoding == "cp1251" - - def test_follow_flags(self): - res = self.response_class("http://example.com/") - fol = res.follow("http://example.com/", flags=["cached", "allowed"]) - assert fol.flags == ["cached", "allowed"] - - def test_follow_all_flags(self): - re = self.response_class("http://www.example.com/") - urls = [ - "http://www.example.com/", - "http://www.example.com/2", - "http://www.example.com/foo", - ] - fol = re.follow_all(urls, flags=["cached", "allowed"]) - for req in fol: - assert req.flags == ["cached", "allowed"] - - def test_follow_all_css(self): - expected = [ - "http://example.com/sample3.html", - "http://example.com/innertag.html", - ] - response = self._links_response() - extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] - assert expected == extracted - - def test_follow_all_css_skip_invalid(self): - expected = [ - "http://example.com/page/1/", - "http://example.com/page/3/", - "http://example.com/page/4/", - ] - response = self._links_response_no_href() - extracted1 = [r.url for r in response.follow_all(css=".pagination a")] - assert expected == extracted1 - extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))] - assert expected == extracted2 - - def test_follow_all_xpath(self): - expected = [ - "http://example.com/sample3.html", - "http://example.com/innertag.html", - ] - response = self._links_response() - extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') - assert expected == [r.url for r in extracted] - - def test_follow_all_xpath_skip_invalid(self): - expected = [ - "http://example.com/page/1/", - "http://example.com/page/3/", - "http://example.com/page/4/", - ] - response = self._links_response_no_href() - extracted1 = [ - r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a') - ] - assert expected == extracted1 - extracted2 = [ - r.url - for r in response.follow_all(response.xpath('//div[@id="pagination"]/a')) - ] - assert expected == extracted2 - - def test_follow_all_too_many_arguments(self): - response = self._links_response() - with pytest.raises( - ValueError, match="Please supply exactly one of the following arguments" - ): - response.follow_all( - css='a[href*="example.com"]', - xpath='//a[contains(@href, "example.com")]', - ) - - def test_json_response(self): - json_body = b"""{"ip": "109.187.217.200"}""" - json_response = self.response_class("http://www.example.com", body=json_body) - assert json_response.json() == {"ip": "109.187.217.200"} - - text_body = b"""text""" - text_response = self.response_class("http://www.example.com", body=text_body) - with pytest.raises( - ValueError, match=r"(Expecting value|Unexpected '<'): line 1" - ): - text_response.json() - - def test_cache_json_response(self): - json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] - for json_body in json_valid_bodies: - json_response = self.response_class( - "http://www.example.com", body=json_body - ) - - with mock.patch("json.loads") as mock_json: - for _ in range(2): - json_response.json() - mock_json.assert_called_once_with(json_body) - - -class TestHtmlResponse(TestTextResponse): - response_class = HtmlResponse - - def test_html_encoding(self): - body = b"""Some page - - Price: \xa3100' - """ - r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, "iso-8859-1", body) - - body = b""" - - Price: \xa3100 - """ - r2 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r2, "iso-8859-1", body) - - # for conflicting declarations headers must take precedence - body = b"""Some page - - Price: \xa3100' - """ - r3 = self.response_class( - "http://www.example.com", - body=body, - headers={"Content-type": ["text/html; charset=iso-8859-1"]}, - ) - self._assert_response_values(r3, "iso-8859-1", body) - - # make sure replace() preserves the encoding of the original response - body = b"New body \xa3" - r4 = r3.replace(body=body) - self._assert_response_values(r4, "iso-8859-1", body) - - def test_html5_meta_charset(self): - body = b"""Some pagebla bla""" - r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, "gb2312", body) - - -class TestXmlResponse(TestTextResponse): - response_class = XmlResponse - - def test_xml_encoding(self): - body = b"" - r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body) - - body = b"""""" - r2 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r2, "iso-8859-1", body) - - # make sure replace() preserves the explicit encoding passed in the __init__ method - body = b"""""" - r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8") - body2 = b"New body" - r4 = r3.replace(body=body2) - self._assert_response_values(r4, "utf-8", body2) - - def test_replace_encoding(self): - # make sure replace() keeps the previous encoding unless overridden explicitly - body = b"""""" - body2 = b"""""" - r5 = self.response_class("http://www.example.com", body=body) - r6 = r5.replace(body=body2) - r7 = r5.replace(body=body2, encoding="utf-8") - self._assert_response_values(r5, "iso-8859-1", body) - self._assert_response_values(r6, "iso-8859-1", body2) - self._assert_response_values(r7, "utf-8", body2) - - def test_selector(self): - body = b'value' - response = self.response_class("http://www.example.com", body=body) - - assert isinstance(response.selector, Selector) - assert response.selector.type == "xml" - assert response.selector is response.selector # property is cached - assert response.selector.response is response - - assert response.selector.xpath("//elem/text()").getall() == ["value"] - - def test_selector_shortcuts(self): - body = b'value' - response = self.response_class("http://www.example.com", body=body) - - assert ( - response.xpath("//elem/text()").getall() - == response.selector.xpath("//elem/text()").getall() - ) - - def test_selector_shortcuts_kwargs(self): - body = b""" - - value - """ - response = self.response_class("http://www.example.com", body=body) - - assert ( - response.xpath( - "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall() - == response.selector.xpath( - "//s:elem/text()", namespaces={"s": "http://scrapy.org"} - ).getall() - ) - - response.selector.register_namespace("s2", "http://scrapy.org") - assert ( - response.xpath( - "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"} - ).getall() - == response.selector.xpath("//s2:elem/text()").getall() - ) - - -class CustomResponse(TextResponse): - attributes = (*TextResponse.attributes, "foo", "bar") - - def __init__(self, *args, **kwargs) -> None: - self.foo = kwargs.pop("foo", None) - self.bar = kwargs.pop("bar", None) - self.lost = kwargs.pop("lost", None) - super().__init__(*args, **kwargs) - - -class TestCustomResponse(TestTextResponse): - response_class = CustomResponse - - def test_copy(self): - super().test_copy() - r1 = self.response_class( - url="https://example.org", - status=200, - foo="foo", - bar="bar", - lost="lost", - ) - r2 = r1.copy() - assert isinstance(r2, self.response_class) - assert r1.foo == r2.foo - assert r1.bar == r2.bar - assert r1.lost == "lost" - assert r2.lost is None - - def test_replace(self): - super().test_replace() - r1 = self.response_class( - url="https://example.org", - status=200, - foo="foo", - bar="bar", - lost="lost", - ) - - r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") - assert isinstance(r2, self.response_class) - assert r1.foo == "foo" - assert r1.bar == "bar" - assert r1.lost == "lost" - assert r2.foo == "new-foo" - assert r2.bar == "new-bar" - assert r2.lost == "new-lost" - - r3 = r1.replace(foo="new-foo", bar="new-bar") - assert isinstance(r3, self.response_class) - assert r1.foo == "foo" - assert r1.bar == "bar" - assert r1.lost == "lost" - assert r3.foo == "new-foo" - assert r3.bar == "new-bar" - assert r3.lost is None - - r4 = r1.replace(foo="new-foo") - assert isinstance(r4, self.response_class) - assert r1.foo == "foo" - assert r1.bar == "bar" - assert r1.lost == "lost" - assert r4.foo == "new-foo" - assert r4.bar == "bar" - assert r4.lost is None - - with pytest.raises( - TypeError, - match=r"__init__\(\) got an unexpected keyword argument 'unknown'", - ): - r1.replace(unknown="unknown") diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py new file mode 100644 index 000000000..4b3fa2302 --- /dev/null +++ b/tests/test_http_response_text.py @@ -0,0 +1,684 @@ +from __future__ import annotations + +import codecs +from unittest import mock + +import pytest + +from scrapy.http import HtmlResponse, TextResponse, XmlResponse +from scrapy.selector import Selector +from scrapy.utils.python import to_unicode +from tests.test_http_response import TestResponse + + +class TestTextResponse(TestResponse): + response_class = TextResponse + + def test_replace(self): + super().test_replace() + r1 = self.response_class( + "http://www.example.com", body="hello", encoding="cp852" + ) + r2 = r1.replace(url="http://www.example.com/other") + r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") + + assert isinstance(r2, self.response_class) + assert r2.url == "http://www.example.com/other" + self._assert_response_encoding(r2, "cp852") + assert r3.url == "http://www.example.com/other" + assert r3._declared_encoding() == "latin1" + + def test_unicode_url(self): + # instantiate with unicode url without encoding (should set default encoding) + resp = self.response_class("http://www.example.com/") + self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) + + # make sure urls are converted to str + resp = self.response_class(url="http://www.example.com/", encoding="utf-8") + assert isinstance(resp.url, str) + + resp = self.response_class( + url="http://www.example.com/price/\xa3", encoding="utf-8" + ) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") + resp = self.response_class( + url="http://www.example.com/price/\xa3", encoding="latin-1" + ) + assert resp.url == "http://www.example.com/price/\xa3" + resp = self.response_class( + "http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}, + ) + assert resp.url == to_unicode(b"http://www.example.com/price/\xc2\xa3") + resp = self.response_class( + "http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + assert resp.url == "http://www.example.com/price/\xa3" + + def test_unicode_body(self): + unicode_string = ( + "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " + "\u0442\u0435\u043a\u0441\u0442" + ) + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="unicode body") + + original_string = unicode_string.encode("cp1251") + r1 = self.response_class( + "http://www.example.com", body=original_string, encoding="cp1251" + ) + + # check response.text + assert isinstance(r1.text, str) + assert r1.text == unicode_string + + def test_encoding(self): + r1 = self.response_class( + "http://www.example.com", + body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}, + ) + r2 = self.response_class( + "http://www.example.com", encoding="utf-8", body="\xa3" + ) + r3 = self.response_class( + "http://www.example.com", + body=b"\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") + r5 = self.response_class( + "http://www.example.com", + body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=None"]}, + ) + r6 = self.response_class( + "http://www.example.com", + body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gb2312"]}, + ) + r7 = self.response_class( + "http://www.example.com", + body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gbk"]}, + ) + r8 = self.response_class( + "http://www.example.com", + body=codecs.BOM_UTF8 + b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=cp1251"]}, + ) + r9 = self.response_class( + "http://www.example.com", + body=b"\x80", + headers={ + "Content-type": [b"application/x-download; filename=\x80dummy.txt"] + }, + ) + + assert r1._headers_encoding() == "utf-8" + assert r2._headers_encoding() is None + assert r2._declared_encoding() == "utf-8" + self._assert_response_encoding(r2, "utf-8") + assert r3._headers_encoding() == "cp1252" + assert r3._declared_encoding() == "cp1252" + assert r4._headers_encoding() is None + assert r5._headers_encoding() is None + assert r8._headers_encoding() == "cp1251" + assert r9._headers_encoding() is None + assert r8._declared_encoding() == "utf-8" + assert r9._declared_encoding() is None + self._assert_response_encoding(r5, "utf-8") + self._assert_response_encoding(r8, "utf-8") + self._assert_response_encoding(r9, "cp1252") + assert r4._body_inferred_encoding() is not None + assert r4._body_inferred_encoding() != "ascii" + self._assert_response_values(r1, "utf-8", "\xa3") + self._assert_response_values(r2, "utf-8", "\xa3") + self._assert_response_values(r3, "iso-8859-1", "\xa3") + self._assert_response_values(r6, "gb18030", "\u2015") + self._assert_response_values(r7, "gb18030", "\u2015") + self._assert_response_values(r9, "cp1252", "€") + + # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies + with pytest.raises(TypeError): + self.response_class("http://www.example.com", body="\xa3") + + def test_declared_encoding_invalid(self): + """Check that unknown declared encodings are ignored""" + r = self.response_class( + "http://www.example.com", + headers={"Content-type": ["text/html; charset=UNKNOWN"]}, + body=b"\xc2\xa3", + ) + assert r._declared_encoding() is None + self._assert_response_values(r, "utf-8", "\xa3") + + def test_utf16(self): + """Test utf-16 because UnicodeDammit is known to have problems with""" + r = self.response_class( + "http://www.example.com", + body=b"\xff\xfeh\x00i\x00", + encoding="utf-16", + ) + self._assert_response_values(r, "utf-16", "hi") + + def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): + r6 = self.response_class( + "http://www.example.com", + headers={"Content-type": ["text/html; charset=utf-8"]}, + body=b"\xef\xbb\xbfWORD\xe3\xab", + ) + assert r6.encoding == "utf-8" + assert r6.text in { + "WORD\ufffd\ufffd", # w3lib < 1.19.0 + "WORD\ufffd", # w3lib >= 1.19.0 + } + + def test_bom_is_removed_from_body(self): + # Inferring encoding from body also cache decoded body as sideeffect, + # this test tries to ensure that calling response.encoding and + # response.text in indistinct order doesn't affect final + # values for encoding and decoded body. + url = "http://example.com" + body = b"\xef\xbb\xbfWORD" + headers = {"Content-type": ["text/html; charset=utf-8"]} + + # Test response without content-type and BOM encoding + response = self.response_class(url, body=body) + assert response.encoding == "utf-8" + assert response.text == "WORD" + response = self.response_class(url, body=body) + assert response.text == "WORD" + assert response.encoding == "utf-8" + + # Body caching sideeffect isn't triggered when encoding is declared in + # content-type header but BOM still need to be removed from decoded + # body + response = self.response_class(url, headers=headers, body=body) + assert response.encoding == "utf-8" + assert response.text == "WORD" + response = self.response_class(url, headers=headers, body=body) + assert response.text == "WORD" + assert response.encoding == "utf-8" + + def test_replace_wrong_encoding(self): + """Test invalid chars are replaced properly""" + r = self.response_class( + "http://www.example.com", + encoding="utf-8", + body=b"PREFIX\xe3\xabSUFFIX", + ) + # XXX: Policy for replacing invalid chars may suffer minor variations + # but it should always contain the unicode replacement char ('\ufffd') + assert "\ufffd" in r.text, repr(r.text) + assert "PREFIX" in r.text, repr(r.text) + assert "SUFFIX" in r.text, repr(r.text) + + # Do not destroy html tags due to encoding bugs + r = self.response_class( + "http://example.com", + encoding="utf-8", + body=b"\xf0value", + ) + assert "value" in r.text, repr(r.text) + + # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse + # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') + # assert '\ufffd' in r.text, repr(r.text) + + def test_selector(self): + body = b"Some page" + response = self.response_class("http://www.example.com", body=body) + + assert isinstance(response.selector, Selector) + assert response.selector.type == "html" + assert response.selector is response.selector # property is cached + assert response.selector.response is response + + assert response.selector.xpath("//title/text()").getall() == ["Some page"] + assert response.selector.css("title::text").getall() == ["Some page"] + assert response.selector.re("Some (.*)") == ["page"] + + def test_selector_shortcuts(self): + body = b"Some page" + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath("//title/text()").getall() + == response.selector.xpath("//title/text()").getall() + ) + assert ( + response.css("title::text").getall() + == response.selector.css("title::text").getall() + ) + + def test_selector_shortcuts_kwargs(self): + body = b'Some page

A nice paragraph.

' + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath( + "normalize-space(//p[@class=$pclass])", pclass="content" + ).getall() + == response.xpath('normalize-space(//p[@class="content"])').getall() + ) + assert ( + response.xpath( + "//title[count(following::p[@class=$pclass])=$pcount]/text()", + pclass="content", + pcount=1, + ).getall() + == response.xpath( + '//title[count(following::p[@class="content"])=1]/text()' + ).getall() + ) + + def test_urljoin_with_base_url(self): + """Test urljoin shortcut which also evaluates base-url through get_base_url().""" + body = b'' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "/test" + ) + absolute = "https://example.net/test" + assert joined == absolute + + body = b'' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "test" + ) + absolute = "http://www.example.com/test" + assert joined == absolute + + body = b'' + joined = self.response_class("http://www.example.com", body=body).urljoin( + "test" + ) + absolute = "http://www.example.com/elsewhere/test" + assert joined == absolute + + def test_follow_selector(self): + resp = self._links_response() + urls = [ + "http://example.com/sample2.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html", + "http://example.com/sample3.html#foo", + "http://www.google.com/something", + "http://example.com/innertag.html", + "http://example.com/page%204.html", + ] + + # select elements + for sellist in [resp.css("a"), resp.xpath("//a")]: + for sel, url in zip(sellist, urls, strict=True): + self._assert_followed_url(sel, url, response=resp) + + # select elements + self._assert_followed_url( + Selector(text='').css("link")[0], + "http://example.com/foo", + response=resp, + ) + + # href attributes should work + for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]: + for sel, url in zip(sellist, urls, strict=True): + self._assert_followed_url(sel, url, response=resp) + + # non-a elements are not supported + with pytest.raises( + ValueError, match="Only and elements are supported" + ): + resp.follow(resp.css("div")[0]) + + def test_follow_selector_list(self): + resp = self._links_response() + with pytest.raises(ValueError, match="SelectorList"): + resp.follow(resp.css("a")) + + def test_follow_selector_invalid(self): + resp = self._links_response() + with pytest.raises(ValueError, match="Unsupported"): + resp.follow(resp.xpath("count(//div)")[0]) + + def test_follow_selector_attribute(self): + resp = self._links_response() + for src in resp.css("img::attr(src)"): + self._assert_followed_url(src, "http://example.com/sample2.jpg") + + def test_follow_selector_no_href(self): + resp = self.response_class( + url="http://example.com", + body=b"click me", + ) + with pytest.raises(ValueError, match="no href"): + resp.follow(resp.css("a")[0]) + + def test_follow_whitespace_selector(self): + resp = self.response_class( + "http://example.com", + body=b"""click me""", + ) + self._assert_followed_url( + resp.css("a")[0], "http://example.com/foo", response=resp + ) + self._assert_followed_url( + resp.css("a::attr(href)")[0], + "http://example.com/foo", + response=resp, + ) + + def test_follow_encoding(self): + resp1 = self.response_class( + "http://example.com", + encoding="utf8", + body='click me'.encode(), + ) + self._assert_followed_url( + resp1.css("a")[0], + "http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82", + response=resp1, + encoding="utf8", + ) + + resp2 = self.response_class( + "http://example.com", + encoding="cp1251", + body='click me'.encode( + "cp1251" + ), + ) + self._assert_followed_url( + resp2.css("a")[0], + "http://example.com/foo?%EF%F0%E8%E2%E5%F2", + response=resp2, + encoding="cp1251", + ) + + def test_follow_flags(self): + res = self.response_class("http://example.com/") + fol = res.follow("http://example.com/", flags=["cached", "allowed"]) + assert fol.flags == ["cached", "allowed"] + + def test_follow_all_flags(self): + re = self.response_class("http://www.example.com/") + urls = [ + "http://www.example.com/", + "http://www.example.com/2", + "http://www.example.com/foo", + ] + fol = re.follow_all(urls, flags=["cached", "allowed"]) + for req in fol: + assert req.flags == ["cached", "allowed"] + + def test_follow_all_css(self): + expected = [ + "http://example.com/sample3.html", + "http://example.com/innertag.html", + ] + response = self._links_response() + extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')] + assert expected == extracted + + def test_follow_all_css_skip_invalid(self): + expected = [ + "http://example.com/page/1/", + "http://example.com/page/3/", + "http://example.com/page/4/", + ] + response = self._links_response_no_href() + extracted1 = [r.url for r in response.follow_all(css=".pagination a")] + assert expected == extracted1 + extracted2 = [r.url for r in response.follow_all(response.css(".pagination a"))] + assert expected == extracted2 + + def test_follow_all_xpath(self): + expected = [ + "http://example.com/sample3.html", + "http://example.com/innertag.html", + ] + response = self._links_response() + extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]') + assert expected == [r.url for r in extracted] + + def test_follow_all_xpath_skip_invalid(self): + expected = [ + "http://example.com/page/1/", + "http://example.com/page/3/", + "http://example.com/page/4/", + ] + response = self._links_response_no_href() + extracted1 = [ + r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a') + ] + assert expected == extracted1 + extracted2 = [ + r.url + for r in response.follow_all(response.xpath('//div[@id="pagination"]/a')) + ] + assert expected == extracted2 + + def test_follow_all_too_many_arguments(self): + response = self._links_response() + with pytest.raises( + ValueError, match="Please supply exactly one of the following arguments" + ): + response.follow_all( + css='a[href*="example.com"]', + xpath='//a[contains(@href, "example.com")]', + ) + + def test_json_response(self): + json_body = b"""{"ip": "109.187.217.200"}""" + json_response = self.response_class("http://www.example.com", body=json_body) + assert json_response.json() == {"ip": "109.187.217.200"} + + text_body = b"""text""" + text_response = self.response_class("http://www.example.com", body=text_body) + with pytest.raises( + ValueError, match=r"(Expecting value|Unexpected '<'): line 1" + ): + text_response.json() + + def test_cache_json_response(self): + json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] + for json_body in json_valid_bodies: + json_response = self.response_class( + "http://www.example.com", body=json_body + ) + + with mock.patch("json.loads") as mock_json: + for _ in range(2): + json_response.json() + mock_json.assert_called_once_with(json_body) + + +class TestHtmlResponse(TestTextResponse): + response_class = HtmlResponse + + def test_html_encoding(self): + body = b"""Some page + + Price: \xa3100' + """ + r1 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r1, "iso-8859-1", body) + + body = b""" + + Price: \xa3100 + """ + r2 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r2, "iso-8859-1", body) + + # for conflicting declarations headers must take precedence + body = b"""Some page + + Price: \xa3100' + """ + r3 = self.response_class( + "http://www.example.com", + body=body, + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + self._assert_response_values(r3, "iso-8859-1", body) + + # make sure replace() preserves the encoding of the original response + body = b"New body \xa3" + r4 = r3.replace(body=body) + self._assert_response_values(r4, "iso-8859-1", body) + + def test_html5_meta_charset(self): + body = b"""Some pagebla bla""" + r1 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r1, "gb2312", body) + + +class TestXmlResponse(TestTextResponse): + response_class = XmlResponse + + def test_xml_encoding(self): + body = b"" + r1 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body) + + body = b"""""" + r2 = self.response_class("http://www.example.com", body=body) + self._assert_response_values(r2, "iso-8859-1", body) + + # make sure replace() preserves the explicit encoding passed in the __init__ method + body = b"""""" + r3 = self.response_class("http://www.example.com", body=body, encoding="utf-8") + body2 = b"New body" + r4 = r3.replace(body=body2) + self._assert_response_values(r4, "utf-8", body2) + + def test_replace_encoding(self): + # make sure replace() keeps the previous encoding unless overridden explicitly + body = b"""""" + body2 = b"""""" + r5 = self.response_class("http://www.example.com", body=body) + r6 = r5.replace(body=body2) + r7 = r5.replace(body=body2, encoding="utf-8") + self._assert_response_values(r5, "iso-8859-1", body) + self._assert_response_values(r6, "iso-8859-1", body2) + self._assert_response_values(r7, "utf-8", body2) + + def test_selector(self): + body = b'value' + response = self.response_class("http://www.example.com", body=body) + + assert isinstance(response.selector, Selector) + assert response.selector.type == "xml" + assert response.selector is response.selector # property is cached + assert response.selector.response is response + + assert response.selector.xpath("//elem/text()").getall() == ["value"] + + def test_selector_shortcuts(self): + body = b'value' + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath("//elem/text()").getall() + == response.selector.xpath("//elem/text()").getall() + ) + + def test_selector_shortcuts_kwargs(self): + body = b""" + + value + """ + response = self.response_class("http://www.example.com", body=body) + + assert ( + response.xpath( + "//s:elem/text()", namespaces={"s": "http://scrapy.org"} + ).getall() + == response.selector.xpath( + "//s:elem/text()", namespaces={"s": "http://scrapy.org"} + ).getall() + ) + + response.selector.register_namespace("s2", "http://scrapy.org") + assert ( + response.xpath( + "//s1:elem/text()", namespaces={"s1": "http://scrapy.org"} + ).getall() + == response.selector.xpath("//s2:elem/text()").getall() + ) + + +class CustomResponse(TextResponse): + attributes = (*TextResponse.attributes, "foo", "bar") + + def __init__(self, *args, **kwargs) -> None: + self.foo = kwargs.pop("foo", None) + self.bar = kwargs.pop("bar", None) + self.lost = kwargs.pop("lost", None) + super().__init__(*args, **kwargs) + + +class TestCustomResponse(TestTextResponse): + response_class = CustomResponse + + def test_copy(self): + super().test_copy() + r1 = self.response_class( + url="https://example.org", + status=200, + foo="foo", + bar="bar", + lost="lost", + ) + r2 = r1.copy() + assert isinstance(r2, self.response_class) + assert r1.foo == r2.foo + assert r1.bar == r2.bar + assert r1.lost == "lost" + assert r2.lost is None + + def test_replace(self): + super().test_replace() + r1 = self.response_class( + url="https://example.org", + status=200, + foo="foo", + bar="bar", + lost="lost", + ) + + r2 = r1.replace(foo="new-foo", bar="new-bar", lost="new-lost") + assert isinstance(r2, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r2.foo == "new-foo" + assert r2.bar == "new-bar" + assert r2.lost == "new-lost" + + r3 = r1.replace(foo="new-foo", bar="new-bar") + assert isinstance(r3, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r3.foo == "new-foo" + assert r3.bar == "new-bar" + assert r3.lost is None + + r4 = r1.replace(foo="new-foo") + assert isinstance(r4, self.response_class) + assert r1.foo == "foo" + assert r1.bar == "bar" + assert r1.lost == "lost" + assert r4.foo == "new-foo" + assert r4.bar == "bar" + assert r4.lost is None + + with pytest.raises( + TypeError, + match=r"__init__\(\) got an unexpected keyword argument 'unknown'", + ): + r1.replace(unknown="unknown") diff --git a/tests/test_item.py b/tests/test_item.py index 4eb37a344..34b054e12 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -48,6 +48,18 @@ class TestItem: with pytest.raises(KeyError): i["field"] + def test_delitem(self): + class TestItem(Item): + name = Field() + + i = TestItem(name="John") + del i["name"] + with pytest.raises(KeyError): + i["name"] + + with pytest.raises(KeyError): + del i["name"] + def test_repr(self): class TestItem(Item): name = Field() diff --git a/tests/test_link.py b/tests/test_link.py index c49e5c090..0eeffe12b 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -55,3 +55,7 @@ class TestLink: def test_bytes_url(self): with pytest.raises(TypeError): Link(b"http://www.example.com/\xc2\xa3") + + def test_eq_non_link(self): + url = "http://example.com" + assert Link(url) != url diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 15d358d2a..063b92dac 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -9,7 +9,7 @@ from w3lib import __version__ as w3lib_version from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link -from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor +from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtractor from tests import get_testdata @@ -837,3 +837,36 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase): def test_link_allowed_is_false_with_missing_url_prefix(self): bad_link = Link("should_have_prefix.example") assert not LxmlLinkExtractor()._link_allowed(bad_link) + + +class TestLxmlParserLinkExtractor: + def test_extract_links(self): + html = b'Link' + response = HtmlResponse("http://example.com/", body=html) + lx = LxmlParserLinkExtractor() + assert lx.extract_links(response) == [ + Link(url="http://example.com/page.html", text="Link", nofollow=False), + ] + + def test_strip_false(self): + # With strip=False, trailing whitespace on a relative href survives urljoin + # and is visible to process_value (safe_url_string cleans it up afterward). + # Here process_value rejects URLs that still carry trailing whitespace, + # demonstrating the difference from strip=True. + def reject_trailing_whitespace(url): + return None if url != url.rstrip() else url + + html = b'Link' + response = HtmlResponse("http://example.com/", body=html) + + lx_strip = LxmlParserLinkExtractor( + strip=True, process=reject_trailing_whitespace + ) + assert lx_strip.extract_links(response) == [ + Link(url="http://example.com/page.html", text="Link", nofollow=False), + ] + + lx_no_strip = LxmlParserLinkExtractor( + strip=False, process=reject_trailing_whitespace + ) + assert lx_no_strip.extract_links(response) == [] diff --git a/tests/test_loader.py b/tests/test_loader.py index 224158e7f..c094d25d8 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -39,7 +39,7 @@ class AttrsNameItem: @dataclasses.dataclass class NameDataClass: - name: list = dataclasses.field(default_factory=list) + name: list[str] = dataclasses.field(default_factory=list) # test item loaders @@ -55,17 +55,6 @@ class ProcessorItemLoader(NameItemLoader): name_in = MapCompose(lambda v: v.title()) -class DefaultedItemLoader(NameItemLoader): - default_input_processor = MapCompose(lambda v: v[:-1]) - - -# test processors -def processor_with_args(value, other=None, loader_context=None): - if "key" in loader_context: - return loader_context["key"] - return value - - class TestBasicItemLoader: def test_add_value_on_unknown_field(self): il = ProcessorItemLoader() diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 9806315b4..360aa613e 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -28,14 +28,14 @@ class TestLogFormatter: self.spider = Spider("default") self.spider.crawler = get_crawler() - def test_crawled_with_referer(self): + def test_crawled_without_referer(self): req = Request("http://www.example.com") res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws["msg"] % logkws["args"] assert logline == "Crawled (200) (referer: None)" - def test_crawled_without_referer(self): + def test_crawled_with_referer(self): req = Request( "http://www.example.com", headers={"referer": "http://example.com"} ) @@ -198,7 +198,7 @@ class TestLogformatterSubclass(TestLogFormatter): self.spider = Spider("default") self.spider.crawler = get_crawler(Spider) - def test_crawled_with_referer(self): + def test_crawled_without_referer(self): req = Request("http://www.example.com") res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) @@ -207,7 +207,7 @@ class TestLogformatterSubclass(TestLogFormatter): logline == "Crawled (200) (referer: None) []" ) - def test_crawled_without_referer(self): + def test_crawled_with_referer(self): req = Request( "http://www.example.com", headers={"referer": "http://example.com"}, diff --git a/tests/test_logstats.py b/tests/test_logstats.py index c2e6d3456..370728e6a 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -5,6 +5,7 @@ import pytest from scrapy.extensions.logstats import LogStats from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider +from tests.utils.decorators import coroutine_test class TestLogStats: @@ -16,8 +17,8 @@ class TestLogStats: self.stats.set_value("response_received_count", 4802) self.stats.set_value("item_scraped_count", 3201) - @pytest.mark.requires_reactor # needs a reactor or an event loop for LogStats.task - def test_stats_calculations(self): + @coroutine_test + async def test_stats_calculations(self): logstats = LogStats.from_crawler(self.crawler) with pytest.raises(AttributeError): diff --git a/tests/test_mail.py b/tests/test_mail.py index c845ba320..e6d65cd10 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -5,11 +5,18 @@ import pytest from twisted.internet import defer from twisted.internet._sslverify import ClientTLSOptions -from scrapy.mail import MailSender +pytestmark = pytest.mark.filterwarnings( + "ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning" +) + +from scrapy.mail import MailSender # noqa: E402 -@pytest.mark.requires_reactor +@pytest.mark.requires_reactor # MailSender requires a reactor class TestMailSender: + def _catch_mail_sent(self, **kwargs): + self.catched_msg = {**kwargs} + def test_send(self): mailsender = MailSender(debug=True) mailsender.send( @@ -88,9 +95,6 @@ class TestMailSender: assert text.get_charset() == Charset("us-ascii") assert attach.get_payload(decode=True) == b"content" - def _catch_mail_sent(self, **kwargs): - self.catched_msg = {**kwargs} - def test_send_utf8(self): subject = "sübjèçt" body = "bödÿ-àéïöñß" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 57736e0ea..3829a35f5 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -2,7 +2,6 @@ import dataclasses import os import random import time -import warnings from abc import ABC, abstractmethod from datetime import datetime from ftplib import FTP @@ -19,6 +18,7 @@ from urllib.parse import urlparse import attr import pytest from itemadapter import ItemAdapter +from twisted.internet.defer import Deferred from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response @@ -31,6 +31,7 @@ from scrapy.pipelines.files import ( S3FilesStore, ) from scrapy.settings import Settings +from scrapy.utils.asyncio import call_later from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.ftp import MockFTPServer @@ -78,6 +79,22 @@ def get_ftp_content_and_delete( return b"".join(ftp_data) +class DeferredFSFilesStore(FSFilesStore): + """A simple store with persist_file() returning a deferred.""" + + def persist_file(self, path, buf, info, meta=None, headers=None): + deferred = Deferred() + # short-hand super() doesn't work in nested functions + parent_persist_file = super().persist_file + + def cb(): + parent_persist_file(path, buf, info, meta=meta, headers=headers) + deferred.callback(None) + + call_later(0.5, cb) + return deferred + + class TestFilesPipeline: def setup_method(self): self.tempdir = mkdtemp() @@ -91,6 +108,15 @@ class TestFilesPipeline: def teardown_method(self): rmtree(self.tempdir) + def test_file_path_query_parameters(self): + file_path = self.pipeline.file_path + + req1 = Request("http://foo.bar/baz.txt?fizz") + assert file_path(req1) == "full/a2b4913a62f65445aeae2bac08cd8c3b41d7195e.txt" + + req2 = Request("http://foo.bar/get_img.foo?file=photo.jpg") + assert file_path(req2) == "full/7fc9461c9fd836515bea6983373097203a7d748e.jpg" + def test_file_path(self): file_path = self.pipeline.file_path assert ( @@ -114,10 +140,10 @@ class TestFilesPipeline: assert ( file_path( Request( - "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg" + "http://www.dfsonline.co.uk/get_prod_image?img=status_0907_mdm.jpg" ) ) - == "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg" + == "full/c67f916ff9d542e822dedf38f9fcb146d1faba78.jpg" ) assert ( file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")) @@ -138,10 +164,10 @@ class TestFilesPipeline: assert ( file_path( Request( - "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha" + "http://www.dfsonline.co.uk/get_prod_image?img=status_0907_mdm.jpg.bohaha" ) ) - == "full/76c00cef2ef669ae65052661f68d451162829507" + == "full/e75f2fa260521b56f6b6a867447b8002d00b5841" ) assert ( file_path( @@ -165,7 +191,7 @@ class TestFilesPipeline: async def test_file_not_expired(self): item_url = "http://example.com/file.pdf" item = _create_item_with_files(item_url) - patchers = [ + with ( mock.patch.object(FilesPipeline, "inc_stats", return_value=True), mock.patch.object( FSFilesStore, @@ -177,22 +203,16 @@ class TestFilesPipeline: "get_media_requests", return_value=[_prepare_request_object(item_url)], ), - ] - for p in patchers: - p.start() - - result = await self.pipeline.process_item(item) + ): + result = await self.pipeline.process_item(item) assert result["files"][0]["checksum"] == "abc" assert result["files"][0]["status"] == "uptodate" - for p in patchers: - p.stop() - @coroutine_test async def test_file_expired(self): item_url = "http://example.com/file2.pdf" item = _create_item_with_files(item_url) - patchers = [ + with ( mock.patch.object( FSFilesStore, "stat_file", @@ -208,22 +228,16 @@ class TestFilesPipeline: return_value=[_prepare_request_object(item_url)], ), mock.patch.object(FilesPipeline, "inc_stats", return_value=True), - ] - for p in patchers: - p.start() - - result = await self.pipeline.process_item(item) + ): + result = await self.pipeline.process_item(item) assert result["files"][0]["checksum"] != "abc" assert result["files"][0]["status"] == "downloaded" - for p in patchers: - p.stop() - @coroutine_test async def test_file_cached(self): item_url = "http://example.com/file3.pdf" item = _create_item_with_files(item_url) - patchers = [ + with ( mock.patch.object(FilesPipeline, "inc_stats", return_value=True), mock.patch.object( FSFilesStore, @@ -239,16 +253,33 @@ class TestFilesPipeline: "get_media_requests", return_value=[_prepare_request_object(item_url, flags=["cached"])], ), - ] - for p in patchers: - p.start() - - result = await self.pipeline.process_item(item) + ): + result = await self.pipeline.process_item(item) assert result["files"][0]["checksum"] != "abc" assert result["files"][0]["status"] == "cached" - for p in patchers: - p.stop() + @coroutine_test + async def test_async_store(self) -> None: + """Test that async persist_file() works and is awaited.""" + + self.pipeline.store = DeferredFSFilesStore(self.tempdir) + item_url = "http://example.com/file.pdf" + item = _create_item_with_files(item_url) + with ( + mock.patch.object(FilesPipeline, "inc_stats", return_value=True), + mock.patch.object( + FilesPipeline, + "get_media_requests", + return_value=[_prepare_request_object(item_url)], + ), + ): + result = await self.pipeline.process_item(item) + assert result["files"][0]["status"] == "downloaded" + assert result["files"][0]["checksum"] + # check that the file was written by persist_file() + path = Path(self.tempdir) / result["files"][0]["path"] + assert path.exists() + assert path.read_bytes() == b"data" def test_file_path_from_item(self): """ @@ -351,11 +382,11 @@ class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin): class FilesPipelineTestDataClass: name: str # default fields - file_urls: list = dataclasses.field(default_factory=list) - files: list = dataclasses.field(default_factory=list) + file_urls: list[str] = dataclasses.field(default_factory=list) + files: list[dict[str, str]] = dataclasses.field(default_factory=list) # overridden fields - custom_file_urls: list = dataclasses.field(default_factory=list) - custom_files: list = dataclasses.field(default_factory=list) + custom_file_urls: list[str] = dataclasses.field(default_factory=list) + custom_files: list[dict[str, str]] = dataclasses.field(default_factory=list) class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin): @@ -691,7 +722,6 @@ class TestGCSFilesStore: store.bucket.get_blob.assert_called_with(expected_blob_path) -@pytest.mark.requires_reactor # needs a reactor for FTPFilesStore class TestFTPFileStore: @inline_callbacks_test def test_persist(self): @@ -729,13 +759,13 @@ class ItemWithFiles(Item): files = Field() -def _create_item_with_files(*files): +def _create_item_with_files(*files: str) -> ItemWithFiles: item = ItemWithFiles() item["file_urls"] = files return item -def _prepare_request_object(item_url, flags=None): +def _prepare_request_object(item_url: str, flags: list[str] | None = None) -> Request: return Request( item_url, meta={"response": Response(item_url, status=200, body=b"data", flags=flags)}, @@ -755,12 +785,10 @@ class TestBuildFromCrawler: class Pipeline(FilesPipeline): pass - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe.store + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe.store def test_has_from_crawler_and_init(self): class Pipeline(FilesPipeline): @@ -774,13 +802,11 @@ class TestBuildFromCrawler: o._from_crawler_called = True return o - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe.store - assert pipe._from_crawler_called + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe.store + assert pipe._from_crawler_called @pytest.mark.parametrize("store", [None, ""]) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 199ec5afa..1b73dd157 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -14,6 +14,7 @@ from itemadapter import ItemAdapter from scrapy.http import Request, Response from scrapy.item import Field, Item +from scrapy.pipelines.files import GCSFilesStore, S3FilesStore from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.utils.test import get_crawler @@ -314,11 +315,11 @@ class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin): class ImagesPipelineTestDataClass: name: str # default fields - image_urls: list = dataclasses.field(default_factory=list) - images: list = dataclasses.field(default_factory=list) + image_urls: list[str] = dataclasses.field(default_factory=list) + images: list[dict[str, str]] = dataclasses.field(default_factory=list) # overridden fields - custom_image_urls: list = dataclasses.field(default_factory=list) - custom_images: list = dataclasses.field(default_factory=list) + custom_image_urls: list[str] = dataclasses.field(default_factory=list) + custom_images: list[dict[str, str]] = dataclasses.field(default_factory=list) class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin): @@ -541,6 +542,44 @@ class TestImagesPipelineCustomSettings: expected_value = settings.get(settings_attr) assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value + def test_images_store_s3_acl_setting_used(self, tmp_path): + old_policy = S3FilesStore.POLICY + + try: + crawler = get_crawler( + None, + { + "IMAGES_STORE": tmp_path, + "IMAGES_STORE_S3_ACL": "public-read", + "FILES_STORE_S3_ACL": "private", + }, + ) + + ImagesPipeline.from_crawler(crawler) + + assert S3FilesStore.POLICY == "public-read" + finally: + S3FilesStore.POLICY = old_policy + + def test_images_store_gcs_acl_setting_used(self, tmp_path): + old_policy = GCSFilesStore.POLICY + + try: + crawler = get_crawler( + None, + { + "IMAGES_STORE": tmp_path, + "IMAGES_STORE_GCS_ACL": "authenticatedRead", + "FILES_STORE_GCS_ACL": "", + }, + ) + + ImagesPipeline.from_crawler(crawler) + + assert GCSFilesStore.POLICY == "authenticatedRead" + finally: + GCSFilesStore.POLICY = old_policy + def _create_image(format_, *a, **kw): buf = io.BytesIO() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 8229d1da1..8ac949da7 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from unittest.mock import MagicMock import pytest @@ -175,10 +174,6 @@ class MockedMediaPipeline(UserDefinedPipeline): super().__init__(*args, crawler=crawler, **kwargs) self._mockcalled = [] - def download(self, request, info): - self._mockcalled.append("download") - return super().download(request, info) - def media_to_download(self, request, info, *, item=None): self._mockcalled.append("media_to_download") if "result" in request.meta: @@ -204,6 +199,11 @@ class MockedMediaPipeline(UserDefinedPipeline): return item +class AsyncMediaDownloadedPipeline(MockedMediaPipeline): + async def media_downloaded(self, response, request, info, *, item=None): + return super().media_downloaded(response, request, info) + + class TestMediaPipeline(TestBaseMediaPipeline): pipeline_class = MockedMediaPipeline @@ -371,6 +371,16 @@ class TestMediaPipeline(TestBaseMediaPipeline): ) +class TestAsyncMediaDownloaded(TestMediaPipeline): + pipeline_class = AsyncMediaDownloadedPipeline + + def test_key_for_pipe(self): + assert ( + self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline") + == "ASYNCMEDIADOWNLOADEDPIPELINE_IMAGES" + ) + + class TestMediaPipelineAllowRedirectSettings: def _assert_request_no3xx(self, pipeline_class, settings): pipe = pipeline_class(crawler=get_crawler(None, settings)) @@ -414,11 +424,9 @@ class TestBuildFromCrawler: class Pipeline(UserDefinedPipeline): pass - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter def test_has_from_crawler_and_init(self): class Pipeline(UserDefinedPipeline): @@ -436,13 +444,11 @@ class TestBuildFromCrawler: o._from_crawler_called = True return o - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe._from_crawler_called - assert pipe._init_called + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe._from_crawler_called + assert pipe._init_called def test_has_from_crawler(self): class Pipeline(UserDefinedPipeline): @@ -456,13 +462,10 @@ class TestBuildFromCrawler: o.store_uri = settings["FILES_STORE"] return o - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - # this and the next assert will fail as MediaPipeline.from_crawler() wasn't called - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe._from_crawler_called + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe._from_crawler_called class MediaFailedFailurePipeline(MockedMediaPipeline): diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 753fcd42e..e65b5bc32 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -248,12 +248,10 @@ class TestPipeline: class TestCustomPipelineManager: - # needs a reactor or an event loop for is_asyncio_available() - # (for ItemPipelineManager.process_item()) - @pytest.mark.requires_reactor - def test_deprecated_process_item_spider_arg(self) -> None: + @coroutine_test + async def test_deprecated_process_item_spider_arg(self) -> None: class CustomPipelineManager(ItemPipelineManager): - def process_item(self, item, spider): # pylint: disable=useless-parent-delegation + def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: # pylint: disable=useless-parent-delegation return super().process_item(item, spider) crawler = get_crawler(DefaultSpider) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 2d42ba791..6c6a6584a 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -1,15 +1,17 @@ import tempfile +from unittest.mock import Mock import pytest import queuelib +from scrapy.core.downloader import Downloader from scrapy.http.request import Request from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue from scrapy.spiders import Spider -from scrapy.squeues import FifoMemoryQueue +from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler -from tests.test_scheduler import MockDownloader, MockEngine +from tests.test_scheduler import MockDownloader class TestPriorityQueue: @@ -74,6 +76,29 @@ class TestPriorityQueue: assert queue.pop().url == req3.url assert not queue.close() + def test_init_prios_with_start_queue(self): + temp_dir = tempfile.mkdtemp() + queue = ScrapyPriorityQueue.from_crawler( + self.crawler, + PickleFifoDiskQueue, + temp_dir, + start_queue_cls=PickleFifoDiskQueue, + ) + req = Request("https://example.org/", meta={"is_start_request": True}) + queue.push(req) + startprios = queue.close() + + queue2 = ScrapyPriorityQueue.from_crawler( + self.crawler, + PickleFifoDiskQueue, + temp_dir, + startprios, + start_queue_cls=PickleFifoDiskQueue, + ) + assert len(queue2) == 1 + assert queue2.pop().url == req.url + queue2.close() + def test_queue_push_pop_priorities(self): temp_dir = tempfile.mkdtemp() queue = ScrapyPriorityQueue.from_crawler( @@ -98,7 +123,7 @@ class TestPriorityQueue: class TestDownloaderAwarePriorityQueue: def setup_method(self): crawler = get_crawler(Spider) - crawler.engine = MockEngine(downloader=MockDownloader()) + crawler.engine = Mock(downloader=MockDownloader()) self.queue = DownloaderAwarePriorityQueue.from_crawler( crawler=crawler, downstream_queue_cls=FifoMemoryQueue, @@ -157,6 +182,81 @@ class TestDownloaderAwarePriorityQueue: assert self.queue.pop().url == req3.url assert self.queue.peek() is None + def test_tie_breaking_rotates_slots(self): + # No active downloads are tracked in the downloader, so every slot has + # the same score and tie-breaking must not starve a slot. + req_a1 = Request("https://example.org/a1") + req_a1.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_b1 = Request("https://example.org/b1") + req_b1.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_a2 = Request("https://example.org/a2") + req_a2.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_b2 = Request("https://example.org/b2") + req_b2.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + + for request in (req_a1, req_b1, req_a2, req_b2): + self.queue.push(request) + + slots = [ + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + ] + + assert slots == ["slot-a", "slot-b", "slot-a", "slot-b"] + + def test_tie_breaking_keeps_rotation_after_selected_slot_is_deleted(self): + # If the selected slot becomes empty, rotation should continue from + # that slot marker to avoid restarting from the smallest slot. + req_a1 = Request("https://example.org/a1") + req_a1.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a2 = Request("https://example.org/a2") + req_a2.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_b1 = Request("https://example.org/b1") + req_b1.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_c1 = Request("https://example.org/c1") + req_c1.meta[Downloader.DOWNLOAD_SLOT] = "slot-c" + + for request in (req_a1, req_a2, req_b1, req_c1): + self.queue.push(request) + + slots = [ + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + ] + + assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"] + + def test_pop_prefers_slot_with_fewer_active_downloads(self): + downloader = self.queue._downloader_interface.downloader + + req_a = Request("https://example.org/a") + req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_b = Request("https://example.org/b") + req_b.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_c = Request("https://example.org/c") + req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c" + + for req in (req_a, req_b, req_c): + self.queue.push(req) + + downloader.increment("slot-a") + downloader.increment("slot-c") + + popped = self.queue.pop() + assert popped.url == req_b.url + + def test_contains(self): + req = Request("https://example.org/") + req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot" + assert "example-slot" not in self.queue + self.queue.push(req) + assert "example-slot" in self.queue + assert "other-slot" not in self.queue + @pytest.mark.parametrize( ("input_", "output"), diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py deleted file mode 100644 index 912075401..000000000 --- a/tests/test_proxy_connect.py +++ /dev/null @@ -1,117 +0,0 @@ -import json -import os -import re -import sys -from pathlib import Path -from subprocess import PIPE, Popen -from urllib.parse import urlsplit, urlunsplit - -import pytest -from testfixtures import LogCapture - -from scrapy.http import Request -from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer -from tests.spiders import SimpleSpider, SingleRequestSpider -from tests.utils.decorators import inline_callbacks_test - - -class MitmProxy: - auth_user = "scrapy" - auth_pass = "scrapy" - - def start(self): - script = """ -import sys -from mitmproxy.tools.main import mitmdump -sys.argv[0] = "mitmdump" -sys.exit(mitmdump()) - """ - cert_path = Path(__file__).parent.resolve() / "keys" - self.proc = Popen( - [ - sys.executable, - "-u", - "-c", - script, - "--listen-host", - "127.0.0.1", - "--listen-port", - "0", - "--proxyauth", - f"{self.auth_user}:{self.auth_pass}", - "--set", - f"confdir={cert_path}", - "--ssl-insecure", - ], - stdout=PIPE, - ) - line = self.proc.stdout.readline().decode("utf-8") - host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1) - return f"http://{self.auth_user}:{self.auth_pass}@{host_port}" - - def stop(self): - self.proc.kill() - self.proc.communicate() - - -def _wrong_credentials(proxy_url): - bad_auth_proxy = list(urlsplit(proxy_url)) - bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@") - return urlunsplit(bad_auth_proxy) - - -@pytest.mark.requires_mitmproxy -class TestProxyConnect: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - def setup_method(self): - self._oldenv = os.environ.copy() - self._proxy = MitmProxy() - proxy_url = self._proxy.start() - os.environ["https_proxy"] = proxy_url - os.environ["http_proxy"] = proxy_url - - def teardown_method(self): - self._proxy.stop() - os.environ = self._oldenv - - @inline_callbacks_test - def test_https_connect_tunnel(self): - crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, log) - - @inline_callbacks_test - def test_https_tunnel_auth_error(self): - os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"]) - crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - # The proxy returns a 407 error code but it does not reach the client; - # he just sees a TunnelError. - self._assert_got_tunnel_error(log) - - @inline_callbacks_test - def test_https_tunnel_without_leak_proxy_authorization_header(self): - request = Request(self.mockserver.url("/echo", is_secure=True)) - crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl(seed=request) - self._assert_got_response_code(200, log) - echo = json.loads(crawler.spider.meta["responses"][0].text) - assert "Proxy-Authorization" not in echo["headers"] - - def _assert_got_response_code(self, code, log): - assert str(log).count(f"Crawled ({code})") == 1 - - def _assert_got_tunnel_error(self, log): - assert "TunnelError" in str(log) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 8d3977452..ee9d2ed51 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -38,8 +38,8 @@ class InjectArgumentsSpiderMiddleware: if request.callback.__name__ == "parse_spider_mw": request.cb_kwargs["from_process_spider_input"] = True - def process_spider_output(self, response, result): - for element in result: + async def process_spider_output(self, response, result): + async for element in result: if ( isinstance(element, Request) and element.callback.__name__ == "parse_spider_mw_2" @@ -102,9 +102,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(kwargs["callback"] == "some_callback") self.crawler.stats.inc_value("boolean_checks", 3) elif response.url.endswith("/general_without"): - self.checks.append( - kwargs == {} # pylint: disable=use-implicit-booleaness-not-comparison - ) + self.checks.append(kwargs == {}) self.crawler.stats.inc_value("boolean_checks") def parse_no_kwargs(self, response): diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index ea7018541..78ff18b15 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,7 +1,7 @@ import pytest from scrapy import Request, Spider -from scrapy.http import FormRequest, JsonRequest +from scrapy.http import JsonRequest from scrapy.utils.request import request_from_dict @@ -67,12 +67,10 @@ class TestRequestSerialization: assert r1.dumps_kwargs == r2.dumps_kwargs def test_request_class(self): - r1 = FormRequest("http://www.example.com") + r1 = CustomRequest("http://www.example.com") self._assert_serializes_ok(r1, spider=self.spider) - r2 = CustomRequest("http://www.example.com") + r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4}) self._assert_serializes_ok(r2, spider=self.spider) - r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4}) - self._assert_serializes_ok(r3, spider=self.spider) def test_callback_serialization(self): r = Request( diff --git a/tests/test_resolver.py b/tests/test_resolver.py new file mode 100644 index 000000000..83fc8693b --- /dev/null +++ b/tests/test_resolver.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from unittest.mock import Mock + +import pytest + +from scrapy.resolver import CachingHostnameResolver, CachingThreadedResolver, dnscache +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + + +@pytest.fixture(autouse=True) +def reset_dnscache(): + original_limit = dnscache.limit + dnscache.clear() + yield + dnscache.clear() + dnscache.limit = original_limit + + +def test_caching_threaded_resolver_dnscache_disabled(): + crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False}) + CachingThreadedResolver.from_crawler(crawler, Mock()) + assert dnscache.limit == 0 + + +@coroutine_test +async def test_caching_threaded_resolver_getHostByName_cache_hit(): + resolver = CachingThreadedResolver(Mock(), cache_size=10, timeout=5.0) + dnscache["example.com"] = "1.2.3.4" + + result = await maybe_deferred_to_future(resolver.getHostByName("example.com")) + assert result == "1.2.3.4" + + +def test_caching_hostname_resolver_dnscache_disabled(): + crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False}) + CachingHostnameResolver.from_crawler(crawler, Mock()) + assert dnscache.limit == 0 + + +def test_caching_hostname_resolver_no_addresses_not_cached(): + def fake_resolve(receiver, *_): + receiver.resolutionBegan(Mock()) + receiver.resolutionComplete() + return receiver + + reactor = Mock() + reactor.nameResolver.resolveHostName.side_effect = fake_resolve + + resolver = CachingHostnameResolver(reactor, cache_size=10) + resolver.resolveHostName(Mock(), "example.com") + + assert "example.com" not in dnscache diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 5b04c7436..42ab29267 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -40,6 +40,9 @@ class TestResponseTypes: retcls = responsetypes.from_content_disposition(source) assert retcls is cls, f"{source} ==> {retcls} != {cls}" + def test_from_content_disposition_no_filename(self): + assert responsetypes.from_content_disposition(b"attachment") is Response + def test_from_content_type(self): mappings = [ ("text/html; charset=UTF-8", HtmlResponse), diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index bc79e3a17..5249736f2 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -6,9 +6,10 @@ from scrapy.robotstxt import ( RerpRobotParser, decode_robotstxt, ) +from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER -def rerp_available(): +def rerp_available() -> bool: # check if robotexclusionrulesparser is installed try: from robotexclusionrulesparser import ( # noqa: PLC0415 @@ -140,13 +141,26 @@ class TestPythonRobotParser(BaseRobotParserTest): def setup_method(self): super()._setUp(PythonRobotParser) + @pytest.mark.skipif( + not STDLIB_IMPROVED_ROBOTFILEPARSER, + reason="RobotFileParser from this Python version does not support length based directives precedence.", + ) def test_length_based_precedence(self): - pytest.skip( - "RobotFileParser does not support length based directives precedence." - ) + super().test_length_based_precedence() + @pytest.mark.skipif( + STDLIB_IMPROVED_ROBOTFILEPARSER, + reason="RobotFileParser from this Python version does not support order based directives precedence.", + ) + def test_order_based_precedence(self): + super().test_order_based_precedence() + + @pytest.mark.skipif( + not STDLIB_IMPROVED_ROBOTFILEPARSER, + reason="RobotFileParser from this Python version does not support wildcards.", + ) def test_allowed_wildcards(self): - pytest.skip("RobotFileParser does not support wildcards.") + super().test_allowed_wildcards() @pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 0d2d8be93..8637de41e 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,33 +1,38 @@ from __future__ import annotations -import shutil -import tempfile import warnings from abc import ABC, abstractmethod from collections import deque -from typing import Any, NamedTuple +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from typing import TYPE_CHECKING, Any, NamedTuple, cast +from unittest.mock import Mock import pytest from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler, Scheduler from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.defer import _schedule_coro +from scrapy.utils.defer import ensure_awaitable from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test, inline_callbacks_test + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + from pathlib import Path class MemoryScheduler(BaseScheduler): paused = False - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - self.queue = deque( + self.queue: deque[Request] = deque( Request(value) if isinstance(value, str) else value for value in getattr(self, "queue", []) ) @@ -54,78 +59,62 @@ class MemoryScheduler(BaseScheduler): self.paused = False -class MockEngine(NamedTuple): - downloader: MockDownloader - - class MockSlot(NamedTuple): active: list[Any] class MockDownloader: - def __init__(self): - self.slots = {} + def __init__(self) -> None: + self.slots: dict[str, MockSlot] = {} - def get_slot_key(self, request): + def get_slot_key(self, request: Request) -> str: if Downloader.DOWNLOAD_SLOT in request.meta: - return request.meta[Downloader.DOWNLOAD_SLOT] + return cast("str", request.meta[Downloader.DOWNLOAD_SLOT]) return urlparse_cached(request).hostname or "" - def increment(self, slot_key): + def increment(self, slot_key: str) -> None: slot = self.slots.setdefault(slot_key, MockSlot(active=[])) slot.active.append(1) - def decrement(self, slot_key): - slot = self.slots.get(slot_key) + def decrement(self, slot_key: str) -> None: + slot = self.slots[slot_key] slot.active.pop() - def close(self): + def close(self) -> None: pass class MockCrawler(Crawler): - def __init__(self, priority_queue_cls, jobdir): + def __init__(self, priority_queue_cls: str, jobdir: Path | None): settings = { "SCHEDULER_DEBUG": False, "SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleLifoDiskQueue", "SCHEDULER_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue", "SCHEDULER_PRIORITY_QUEUE": priority_queue_cls, - "JOBDIR": jobdir, + "JOBDIR": str(jobdir) if jobdir is not None else None, "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", } super().__init__(Spider, settings) - self.engine = MockEngine(downloader=MockDownloader()) + self.engine = Mock(downloader=MockDownloader()) self.stats = load_object(self.settings["STATS_CLASS"])(self) -# needs a reactor or an event loop for is_asyncio_available() -# (for _schedule_coro()) -@pytest.mark.requires_reactor -class SchedulerHandler(ABC): - jobdir = None - - @property - @abstractmethod - def priority_queue_cls(self) -> str: - raise NotImplementedError - - def create_scheduler(self): - self.mock_crawler = MockCrawler(self.priority_queue_cls, self.jobdir) - self.scheduler = Scheduler.from_crawler(self.mock_crawler) - self.spider = Spider(name="spider") - self.scheduler.open(self.spider) - - def close_scheduler(self): - self.scheduler.close("finished") - _schedule_coro(self.mock_crawler.stop_async()) - self.mock_crawler.engine.downloader.close() - - def setup_method(self): - self.create_scheduler() - - def teardown_method(self): - self.close_scheduler() +@asynccontextmanager +async def create_scheduler( + priority_queue_cls: str, jobdir: Path | None +) -> AsyncGenerator[Scheduler]: + mock_crawler = MockCrawler(priority_queue_cls, jobdir) + scheduler = Scheduler.from_crawler(mock_crawler) + spider = Spider(name="spider") + await ensure_awaitable(scheduler.open(spider)) + try: + yield scheduler + finally: + await ensure_awaitable(scheduler.close("finished")) + await mock_crawler.stop_async() + assert mock_crawler.engine + mock_crawler.engine.downloader.close() _PRIORITIES = [ @@ -140,99 +129,118 @@ _PRIORITIES = [ _URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"} -class TestSchedulerInMemoryBase(SchedulerHandler): - def test_length(self): - assert not self.scheduler.has_pending_requests() - assert len(self.scheduler) == 0 +class TestSchedulerBase(ABC): + @property + @abstractmethod + def priority_queue_cls(self) -> str: + raise NotImplementedError - for url in _URLS: - self.scheduler.enqueue_request(Request(url)) + @pytest.fixture + def jobdir(self) -> Path | None: + return None - assert self.scheduler.has_pending_requests() - assert len(self.scheduler) == len(_URLS) + def create_scheduler( + self, jobdir: Path | None + ) -> AbstractAsyncContextManager[Scheduler]: + return create_scheduler(self.priority_queue_cls, jobdir) - def test_dequeue(self): - for url in _URLS: - self.scheduler.enqueue_request(Request(url)) + # TODO: unify test methods using "reopen" like in DownloaderAwareSchedulerTestMixin - urls = set() - while self.scheduler.has_pending_requests(): - urls.add(self.scheduler.next_request().url) + +class TestSchedulerInMemoryBase(TestSchedulerBase): + @coroutine_test + async def test_length(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + assert not scheduler.has_pending_requests() + assert len(scheduler) == 0 + + for url in _URLS: + scheduler.enqueue_request(Request(url)) + + assert scheduler.has_pending_requests() + assert len(scheduler) == len(_URLS) + + @coroutine_test + async def test_dequeue(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + for url in _URLS: + scheduler.enqueue_request(Request(url)) + + urls = set() + while scheduler.has_pending_requests(): + request = scheduler.next_request() + assert request is not None + urls.add(request.url) assert urls == _URLS - def test_dequeue_priorities(self): - for url, priority in _PRIORITIES: - self.scheduler.enqueue_request(Request(url, priority=priority)) + @coroutine_test + async def test_dequeue_priorities(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + for url, priority in _PRIORITIES: + scheduler.enqueue_request(Request(url, priority=priority)) - priorities = [] - while self.scheduler.has_pending_requests(): - priorities.append(self.scheduler.next_request().priority) + priorities = [] + while scheduler.has_pending_requests(): + request = scheduler.next_request() + assert request is not None + priorities.append(request.priority) assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) -class TestSchedulerOnDiskBase(SchedulerHandler): - def setup_method(self): - self.jobdir = tempfile.mkdtemp() - self.create_scheduler() +class TestSchedulerOnDiskBase(TestSchedulerBase): + @pytest.fixture + def jobdir(self, tmp_path: Path) -> Path | None: + return tmp_path - def teardown_method(self): - self.close_scheduler() + @coroutine_test + async def test_length(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + assert not scheduler.has_pending_requests() + assert len(scheduler) == 0 + for url in _URLS: + scheduler.enqueue_request(Request(url)) - shutil.rmtree(self.jobdir) - self.jobdir = None + async with self.create_scheduler(jobdir) as scheduler: + assert scheduler.has_pending_requests() + assert len(scheduler) == len(_URLS) - def test_length(self): - assert not self.scheduler.has_pending_requests() - assert len(self.scheduler) == 0 - - for url in _URLS: - self.scheduler.enqueue_request(Request(url)) - - self.close_scheduler() - self.create_scheduler() - - assert self.scheduler.has_pending_requests() - assert len(self.scheduler) == len(_URLS) - - def test_dequeue(self): - for url in _URLS: - self.scheduler.enqueue_request(Request(url)) - - self.close_scheduler() - self.create_scheduler() + @coroutine_test + async def test_dequeue(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + for url in _URLS: + scheduler.enqueue_request(Request(url)) urls = set() - while self.scheduler.has_pending_requests(): - urls.add(self.scheduler.next_request().url) - + async with self.create_scheduler(jobdir) as scheduler: + while scheduler.has_pending_requests(): + request = scheduler.next_request() + assert request is not None + urls.add(request.url) assert urls == _URLS - def test_dequeue_priorities(self): - for url, priority in _PRIORITIES: - self.scheduler.enqueue_request(Request(url, priority=priority)) - - self.close_scheduler() - self.create_scheduler() + @coroutine_test + async def test_dequeue_priorities(self, jobdir: Path | None) -> None: + async with self.create_scheduler(jobdir) as scheduler: + for url, priority in _PRIORITIES: + scheduler.enqueue_request(Request(url, priority=priority)) priorities = [] - while self.scheduler.has_pending_requests(): - priorities.append(self.scheduler.next_request().priority) - + async with self.create_scheduler(jobdir) as scheduler: + while scheduler.has_pending_requests(): + request = scheduler.next_request() + assert request is not None + priorities.append(request.priority) assert priorities == sorted([x[1] for x in _PRIORITIES], key=lambda x: -x) class TestSchedulerInMemory(TestSchedulerInMemoryBase): - @property - def priority_queue_cls(self) -> str: - return "scrapy.pqueues.ScrapyPriorityQueue" + priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" class TestSchedulerOnDisk(TestSchedulerOnDiskBase): - @property - def priority_queue_cls(self) -> str: - return "scrapy.pqueues.ScrapyPriorityQueue" + priority_queue_cls = "scrapy.pqueues.ScrapyPriorityQueue" _URLS_WITH_SLOTS = [ @@ -246,39 +254,25 @@ _URLS_WITH_SLOTS = [ class TestMigration: - # needs a reactor or an event loop for is_asyncio_available() - # (for _schedule_coro()) - @pytest.mark.requires_reactor - def test_migration(self, tmpdir): - class PrevSchedulerHandler(SchedulerHandler): - jobdir = tmpdir + @coroutine_test + async def test_migration(self, tmp_path: Path) -> None: + async with create_scheduler( + "scrapy.pqueues.ScrapyPriorityQueue", tmp_path + ) as prev_scheduler: + for url in _URLS: + prev_scheduler.enqueue_request(Request(url)) - @property - def priority_queue_cls(self) -> str: - return "scrapy.pqueues.ScrapyPriorityQueue" - - class NextSchedulerHandler(SchedulerHandler): - jobdir = tmpdir - - @property - def priority_queue_cls(self) -> str: - return "scrapy.pqueues.DownloaderAwarePriorityQueue" - - prev_scheduler_handler = PrevSchedulerHandler() - prev_scheduler_handler.create_scheduler() - for url in _URLS: - prev_scheduler_handler.scheduler.enqueue_request(Request(url)) - prev_scheduler_handler.close_scheduler() - - next_scheduler_handler = NextSchedulerHandler() with pytest.raises( ValueError, match="DownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict", ): - next_scheduler_handler.create_scheduler() + async with create_scheduler( + "scrapy.pqueues.DownloaderAwarePriorityQueue", tmp_path + ): + pass -def _is_scheduling_fair(enqueued_slots, dequeued_slots): +def _is_scheduling_fair(enqueued_slots: list[str], dequeued_slots: list[str]) -> bool: """ We enqueued same number of requests for every slot. Assert correct order, e.g. @@ -303,39 +297,49 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): return True -class DownloaderAwareSchedulerTestMixin: +class DownloaderAwareSchedulerTestMixin(TestSchedulerBase): reopen = False + priority_queue_cls = "scrapy.pqueues.DownloaderAwarePriorityQueue" - @property - def priority_queue_cls(self) -> str: - return "scrapy.pqueues.DownloaderAwarePriorityQueue" + @coroutine_test + async def test_logic(self, jobdir: Path | None) -> None: + def _setup(scheduler: Scheduler) -> None: + for url, slot in _URLS_WITH_SLOTS: + request = Request(url) + request.meta[Downloader.DOWNLOAD_SLOT] = slot + scheduler.enqueue_request(request) - def test_logic(self): - for url, slot in _URLS_WITH_SLOTS: - request = Request(url) - request.meta[Downloader.DOWNLOAD_SLOT] = slot - self.scheduler.enqueue_request(request) + def _assert(scheduler: Scheduler) -> None: + dequeued_slots: list[str] = [] + requests: list[Request] = [] + assert scheduler.crawler + assert scheduler.crawler.engine + downloader = scheduler.crawler.engine.downloader + assert isinstance(downloader, MockDownloader) + while scheduler.has_pending_requests(): + request = scheduler.next_request() + assert request is not None + slot = downloader.get_slot_key(request) + dequeued_slots.append(slot) + downloader.increment(slot) + requests.append(request) + + for request in requests: + slot = downloader.get_slot_key(request) + downloader.decrement(slot) + + assert _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots) + assert sum(len(s.active) for s in downloader.slots.values()) == 0 if self.reopen: - self.close_scheduler() - self.create_scheduler() - - dequeued_slots = [] - requests = [] - downloader = self.mock_crawler.engine.downloader - while self.scheduler.has_pending_requests(): - request = self.scheduler.next_request() - slot = downloader.get_slot_key(request) - dequeued_slots.append(slot) - downloader.increment(slot) - requests.append(request) - - for request in requests: - slot = downloader.get_slot_key(request) - downloader.decrement(slot) - - assert _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots) - assert sum(len(s.active) for s in downloader.slots.values()) == 0 + async with self.create_scheduler(jobdir) as scheduler: + _setup(scheduler) + async with self.create_scheduler(jobdir) as scheduler: + _assert(scheduler) + else: + async with self.create_scheduler(jobdir) as scheduler: + _setup(scheduler) + _assert(scheduler) class TestSchedulerWithDownloaderAwareInMemory( @@ -393,7 +397,11 @@ class TestIncompatibility: def test_incompatibility(self): with warnings.catch_warnings(): - warnings.filterwarnings("ignore") + warnings.filterwarnings( + "ignore", + category=ScrapyDeprecationWarning, + message="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", + ) with pytest.raises( ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP" ): diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 176fefbbf..08acacae7 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -41,10 +41,10 @@ class MinimalScheduler: class SimpleScheduler(MinimalScheduler): - def open(self, spider: Spider) -> defer.Deferred: + def open(self, spider: Spider) -> defer.Deferred[str]: return defer.succeed("open") - def close(self, reason: str) -> defer.Deferred: + def close(self, reason: str) -> defer.Deferred[str]: return defer.succeed("close") def __len__(self) -> int: @@ -104,7 +104,7 @@ class TestMinimalScheduler(InterfaceCheckMixin): for url in URLS: assert self.scheduler.enqueue_request(Request(url)) assert not self.scheduler.enqueue_request(Request(url)) - assert self.scheduler.has_pending_requests + assert self.scheduler.has_pending_requests() dequeued = [] while self.scheduler.has_pending_requests(): diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py index 4c365859b..4647e1041 100644 --- a/tests/test_scrapy__getattr__.py +++ b/tests/test_scrapy__getattr__.py @@ -1,15 +1,18 @@ -import warnings +from __future__ import annotations + +import pytest + +from scrapy.exceptions import ScrapyDeprecationWarning -def test_deprecated_concurrent_requests_per_ip_attribute(): - with warnings.catch_warnings(record=True) as warns: +def test_deprecated_concurrent_requests_per_ip_attribute() -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match=r"scrapy\.settings\.default_settings\.CONCURRENT_REQUESTS_PER_IP attribute is deprecated", + ): from scrapy.settings.default_settings import ( # noqa: PLC0415 CONCURRENT_REQUESTS_PER_IP, ) - assert CONCURRENT_REQUESTS_PER_IP is not None - assert isinstance(CONCURRENT_REQUESTS_PER_IP, int) - assert ( - "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead." - in warns[0].message.args - ) + assert CONCURRENT_REQUESTS_PER_IP is not None + assert isinstance(CONCURRENT_REQUESTS_PER_IP, int) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 4436f03ba..3282b0591 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,13 +1,13 @@ -# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison +# pylint: disable=unsubscriptable-object,unsupported-membership-test # (too many false positives) import logging -import warnings from unittest import mock import pytest from scrapy.core.downloader.handlers.file import FileDownloadHandler +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import ( SETTINGS_PRIORITIES, BaseSettings, @@ -384,15 +384,15 @@ class TestBaseSettings: "TEST_STRING": "a string", "TEST_LIST": [1, 2], "TEST_BOOLEAN": False, - "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"), - "TEST": BaseSettings({1: 10, 3: 30}, "default"), - "HASNOBASE": BaseSettings({3: 3000}, "default"), + "TEST_BASE": BaseSettings({"foo": 1, "bar": 2}, "project"), + "TEST": BaseSettings({"foo": 10, "baz": 30}, "default"), + "HASNOBASE": BaseSettings({"baz": 3000}, "default"), } ) assert s.copy_to_dict() == { - "HASNOBASE": {3: 3000}, - "TEST": {1: 10, 3: 30}, - "TEST_BASE": {1: 1, 2: 2}, + "HASNOBASE": {"baz": 3000}, + "TEST": {"foo": 10, "baz": 30}, + "TEST_BASE": {"foo": 1, "bar": 2}, "TEST_LIST": [1, 2], "TEST_BOOLEAN": False, "TEST_STRING": "a string", @@ -410,7 +410,45 @@ class TestBaseSettings: assert frozencopy.frozen assert frozencopy is not self.settings - def test_getwithbase_override_none_by_type(self): + def test_getwithbase_for_dotted_keys(self): + settings = BaseSettings( + { + "FEED_EXPORTERS_BASE": BaseSettings({"json": "foo"}), + "FEED_EXPORTERS": BaseSettings({"csv.gz": "bar"}), + } + ) + value = settings.getwithbase("FEED_EXPORTERS") + assert isinstance(value, BaseSettings) + assert dict(value) == { + "json": "foo", + "csv.gz": "bar", + } + + @pytest.mark.parametrize( + ("key", "exception"), + [ + pytest.param(1, TypeError, id="type-error"), + pytest.param("foo", ValueError, id="value-error"), + pytest.param("csv.gz", NameError, id="name-error"), + ], + ) + def test_get_component_priority_dict_with_base_handles_load_object_exceptions( + self, key, exception + ): + with pytest.raises(exception): + load_object(key) + + settings = BaseSettings( + { + "FOO": BaseSettings({key: 1}), + } + ) + value = settings.get_component_priority_dict_with_base("FOO") + + assert isinstance(value, BaseSettings) + assert dict(value) == {key: 1} + + def test_get_component_priority_dict_with_base_override_none_by_type(self): settings = BaseSettings() setting_names = set() for k, v in scrapy_default_settings.__dict__.items(): @@ -426,10 +464,10 @@ class TestBaseSettings: load_object(import_path): None for import_path in v } for setting_name in setting_names: - value = settings.getwithbase(setting_name) + value = settings.get_component_priority_dict_with_base(setting_name) assert not dict(value) - def test_getwithbase_override_value_by_type(self): + def test_get_component_priority_dict_with_base_override_value_by_type(self): settings = BaseSettings() setting_names = set() value = 0 @@ -446,7 +484,10 @@ class TestBaseSettings: load_object(import_path): value for import_path in v } for setting_name in setting_names: - assert settings.getwithbase(setting_name) == settings[setting_name] + assert ( + settings.get_component_priority_dict_with_base(setting_name) + == settings[setting_name] + ) def test_getwithbase_for_non_component_priority_dicts(self): settings = BaseSettings() @@ -465,7 +506,9 @@ class TestBaseSettings: assert isinstance(value, BaseSettings) assert dict(value) == expected - def test_getwithbase_warns_on_duplicate_import_paths(self, caplog): + def test_get_component_priority_dict_with_base_warns_on_duplicate_import_paths( + self, caplog + ): settings = BaseSettings() settings["FOO"] = BaseSettings( { @@ -474,26 +517,78 @@ class TestBaseSettings: } ) with caplog.at_level(logging.WARNING): - value = settings.getwithbase("FOO") + value = settings.get_component_priority_dict_with_base("FOO") assert isinstance(value, BaseSettings) assert dict(value) == {"scrapy.http.Request": 2} assert caplog.records, "Expected a warning to be logged" msg = caplog.records[0].message assert "scrapy.http.request.Request" in msg - def test_getwithbase_warns_on_duplicate_mixed_type_and_path(self, caplog): + def test_get_component_priority_dict_with_base_warns_on_duplicate_mixed_type_and_path( + self, caplog + ): settings = BaseSettings() settings["FOO"] = BaseSettings( {Component1: 1, "tests.test_settings.Component1": 2} ) with caplog.at_level(logging.WARNING): - value = settings.getwithbase("FOO") + value = settings.get_component_priority_dict_with_base("FOO") assert isinstance(value, BaseSettings) assert dict(value) == {"tests.test_settings.Component1": 2} assert caplog.records, "Expected a warning to be logged" msg = caplog.records[0].message assert "tests.test_settings.Component1" in msg + def test_getdictorlist(self): + settings = BaseSettings() + + # No value and no default → {} + assert settings.getdictorlist("MISSING") == {} + + # String: valid JSON dict + settings.set("S_DICT_STR", '{"key": "val"}') + assert settings.getdictorlist("S_DICT_STR") == {"key": "val"} + + # String: valid JSON list + settings.set("S_LIST_STR", '["a", "b"]') + assert settings.getdictorlist("S_LIST_STR") == ["a", "b"] + + # String: invalid JSON → comma-split fallback + settings.set("S_CSV", "a,b,c") + assert settings.getdictorlist("S_CSV") == ["a", "b", "c"] + + # String: valid JSON but not dict or list → ValueError caught → comma-split + settings.set("S_JSON_NUMBER", "123") + assert settings.getdictorlist("S_JSON_NUMBER") == ["123"] + + # Tuple → list + settings.set("S_TUPLE", ("x", "y")) + assert settings.getdictorlist("S_TUPLE") == ["x", "y"] + + # Unsupported type → raises ValueError + settings.set("S_INT", 42) + with pytest.raises(ValueError, match="must be a dict, list, tuple, or string"): + settings.getdictorlist("S_INT") + + # Dict value → deepcopy returned + settings.set("S_DICT", {"key": "val"}) + assert settings.getdictorlist("S_DICT") == {"key": "val"} + + # List value → deepcopy returned + settings.set("S_LIST", ["a", "b"]) + assert settings.getdictorlist("S_LIST") == ["a", "b"] + + def test_repr_pretty_(self): + settings = BaseSettings({"key": "value"}) + mock_p = mock.Mock() + + settings._repr_pretty_(mock_p, cycle=False) + assert mock_p.text.call_count == 1 + + mock_p.reset_mock() + settings._repr_pretty_(mock_p, cycle=True) + mock_p.text.assert_called_once_with(repr(settings)) + def test_getwithbase_invalid_setting_name(self): settings = BaseSettings() with pytest.raises( @@ -501,6 +596,13 @@ class TestBaseSettings: ): settings.getwithbase(123) + def test_get_component_priority_dict_with_base_invalid_setting_name(self): + settings = BaseSettings() + with pytest.raises( + ValueError, match="Base setting key must be a string, got 123" + ): + settings.get_component_priority_dict_with_base(123) + class TestSettings: def setup_method(self): @@ -658,15 +760,22 @@ def test_remove_from_list(before, name, item, after): assert settings.getpriority(name) == expected_settings.getpriority(name) -def test_deprecated_concurrent_requests_per_ip_setting(): - with warnings.catch_warnings(record=True) as warns: - settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) - settings.get("CONCURRENT_REQUESTS_PER_IP") +def test_deprecated_dns_resolver_setting(): + settings = Settings() + with pytest.warns( + ScrapyDeprecationWarning, + match="The DNS_RESOLVER setting is deprecated", + ): + settings.get("DNS_RESOLVER") - assert ( - str(warns[0].message) - == "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead." - ) + +def test_deprecated_concurrent_requests_per_ip_setting(): + settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) + with pytest.warns( + ScrapyDeprecationWarning, + match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", + ): + settings.get("CONCURRENT_REQUESTS_PER_IP") class Component1: diff --git a/tests/test_signalmanager.py b/tests/test_signalmanager.py new file mode 100644 index 000000000..ce4d97adb --- /dev/null +++ b/tests/test_signalmanager.py @@ -0,0 +1,21 @@ +from scrapy.signalmanager import SignalManager + + +class TestSignalManager: + def test_disconnect_all(self): + signal = object() + sender = object() + sm = SignalManager(sender) + + calls = [] + + def handler(): + calls.append(1) + + sm.connect(handler, signal) + sm.send_catch_log(signal) + assert calls == [1] + + sm.disconnect_all(signal) + sm.send_catch_log(signal) + assert calls == [1] # handler no longer called after disconnect_all diff --git a/tests/test_spider.py b/tests/test_spider.py index 68d4f85db..526cc8f23 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,36 +1,19 @@ from __future__ import annotations -import gzip -import re -import warnings -from datetime import datetime -from io import BytesIO -from logging import ERROR, WARNING -from pathlib import Path from typing import Any from unittest import mock import pytest from testfixtures import LogCapture -from w3lib.url import safe_url_string from scrapy import signals from scrapy.crawler import Crawler -from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse -from scrapy.linkextractors import LinkExtractor +from scrapy.http import Response, TextResponse, XmlResponse from scrapy.settings import Settings -from scrapy.spiders import ( - CrawlSpider, - CSVFeedSpider, - Rule, - SitemapSpider, - Spider, - XMLFeedSpider, -) -from scrapy.spiders.init import InitSpider +from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider from scrapy.utils.test import get_crawler, get_reactor_settings -from tests import get_testdata, tests_datadir -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests import get_testdata +from tests.utils.decorators import inline_callbacks_test class TestSpider: @@ -39,7 +22,7 @@ class TestSpider: def test_base_spider(self): spider = self.spider_class("example.com") assert spider.name == "example.com" - assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison + assert spider.start_urls == [] def test_spider_args(self): """``__init__`` method arguments are assigned to spider attributes""" @@ -138,27 +121,6 @@ class TestSpider: mock_logger.log.assert_called_once_with("INFO", "test log msg") -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestInitSpider(TestSpider): - spider_class = InitSpider - - @coroutine_test - async def test_start_urls(self): - responses = [] - - class TestSpider(self.spider_class): - name = "test" - start_urls = ["data:,"] - - async def parse(self, response): - responses.append(response) - - crawler = get_crawler(TestSpider) - await crawler.crawl_async() - assert len(responses) == 1 - assert responses[0].url == "data:," - - class TestXMLFeedSpider(TestSpider): spider_class = XMLFeedSpider @@ -229,629 +191,6 @@ class TestCSVFeedSpider(TestSpider): assert len(rows) == 4 -class TestCrawlSpider(TestSpider): - test_body = b"""Page title - -

Item 12

-
-

About us

-
- - """ - spider_class = CrawlSpider - - def test_rule_without_link_extractor(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(),) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ] - - def test_process_links(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) - - def dummy_process_links(self, links): - return links - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ] - - def test_process_links_filter(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_links="filter_process_links"),) - _test_regex = re.compile("nofollow") - - def filter_process_links(self, links): - return [link for link in links if not self._test_regex.search(link.url)] - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 2 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - ] - - def test_process_links_generator(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) - - def dummy_process_links(self, links): - yield from links - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ] - - def test_process_request(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - def process_request_change_domain(request, response): - return request.replace(url=request.url.replace(".org", ".com")) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = ( - Rule(LinkExtractor(), process_request=process_request_change_domain), - ) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.com/somepage/item/12.html", - "http://example.com/about.html", - "http://example.com/nofollow.html", - ] - - def test_process_request_with_response(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - def process_request_meta_response_class(request, response): - request.meta["response_class"] = response.__class__.__name__ - return request - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = ( - Rule( - LinkExtractor(), process_request=process_request_meta_response_class - ), - ) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ] - assert [r.meta["response_class"] for r in output] == [ - "HtmlResponse", - "HtmlResponse", - "HtmlResponse", - ] - - def test_process_request_instance_method(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = (Rule(LinkExtractor(), process_request="process_request_upper"),) - - def process_request_upper(self, request, response): - return request.replace(url=request.url.upper()) - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), - safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), - safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), - ] - - def test_process_request_instance_method_with_response(self): - response = HtmlResponse( - "http://example.org/somepage/index.html", body=self.test_body - ) - - class _CrawlSpider(self.spider_class): - name = "test" - allowed_domains = ["example.org"] - rules = ( - Rule( - LinkExtractor(), - process_request="process_request_meta_response_class", - ), - ) - - def process_request_meta_response_class(self, request, response): - request.meta["response_class"] = response.__class__.__name__ - return request - - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - assert len(output) == 3 - assert all(isinstance(r, Request) for r in output) - assert [r.url for r in output] == [ - "http://example.org/somepage/item/12.html", - "http://example.org/about.html", - "http://example.org/nofollow.html", - ] - assert [r.meta["response_class"] for r in output] == [ - "HtmlResponse", - "HtmlResponse", - "HtmlResponse", - ] - - def test_follow_links_attribute_population(self): - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - assert hasattr(spider, "_follow_links") - assert spider._follow_links - - settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} - crawler = get_crawler(settings_dict=settings_dict) - spider = self.spider_class.from_crawler(crawler, "example.com") - assert hasattr(spider, "_follow_links") - assert not spider._follow_links - - @inline_callbacks_test - def test_start_url(self): - class TestSpider(self.spider_class): - name = "test" - start_url = "https://www.example.com" - - crawler = get_crawler(TestSpider) - with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: - yield crawler.crawl() - assert "Error while reading start items and requests" in str(log) - assert "did you miss an 's'?" in str(log) - - def test_parse_response_use(self): - class _CrawlSpider(CrawlSpider): - name = "test" - start_urls = "https://www.example.com" - _follow_links = False - - with warnings.catch_warnings(record=True) as w: - spider = _CrawlSpider() - assert len(w) == 0 - spider._parse_response( - TextResponse(spider.start_urls, body=b""), None, None - ) - assert len(w) == 1 - - def test_parse_response_override(self): - class _CrawlSpider(CrawlSpider): - def _parse_response(self, response, callback, cb_kwargs, follow=True): - pass - - name = "test" - start_urls = "https://www.example.com" - _follow_links = False - - with warnings.catch_warnings(record=True) as w: - assert len(w) == 0 - spider = _CrawlSpider() - assert len(w) == 1 - spider._parse_response( - TextResponse(spider.start_urls, body=b""), None, None - ) - assert len(w) == 1 - - def test_parse_with_rules(self): - class _CrawlSpider(CrawlSpider): - name = "test" - start_urls = "https://www.example.com" - - with warnings.catch_warnings(record=True) as w: - spider = _CrawlSpider() - spider.parse_with_rules( - TextResponse(spider.start_urls, body=b""), None, None - ) - assert len(w) == 0 - - -class TestSitemapSpider(TestSpider): - spider_class = SitemapSpider - - BODY = b"SITEMAP" - f = BytesIO() - g = gzip.GzipFile(fileobj=f, mode="w+b") - g.write(BODY) - g.close() - GZBODY = f.getvalue() - - def assertSitemapBody(self, response: Response, body: bytes | None) -> None: - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - assert spider._get_sitemap_body(response) == body - - def test_get_sitemap_body(self): - r = XmlResponse(url="http://www.example.com/", body=self.BODY) - self.assertSitemapBody(r, self.BODY) - - r = HtmlResponse(url="http://www.example.com/", body=self.BODY) - self.assertSitemapBody(r, None) - - r = Response(url="http://www.example.com/favicon.ico", body=self.BODY) - self.assertSitemapBody(r, None) - - def test_get_sitemap_body_gzip_headers(self): - r = Response( - url="http://www.example.com/sitemap", - body=self.GZBODY, - headers={"content-type": "application/gzip"}, - request=Request("http://www.example.com/sitemap"), - ) - self.assertSitemapBody(r, self.BODY) - - def test_get_sitemap_body_xml_url(self): - r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY) - self.assertSitemapBody(r, self.BODY) - - def test_get_sitemap_body_xml_url_compressed(self): - r = Response( - url="http://www.example.com/sitemap.xml.gz", - body=self.GZBODY, - request=Request("http://www.example.com/sitemap"), - ) - self.assertSitemapBody(r, self.BODY) - - # .xml.gz but body decoded by HttpCompression middleware already - r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) - self.assertSitemapBody(r, self.BODY) - - def test_get_sitemap_urls_from_robotstxt(self): - robots = b"""# Sitemap files -Sitemap: http://example.com/sitemap.xml -Sitemap: http://example.com/sitemap-product-index.xml -Sitemap: HTTP://example.com/sitemap-uppercase.xml -Sitemap: /sitemap-relative-url.xml -""" - - r = TextResponse(url="http://www.example.com/robots.txt", body=robots) - spider = self.spider_class("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://example.com/sitemap.xml", - "http://example.com/sitemap-product-index.xml", - "http://example.com/sitemap-uppercase.xml", - "http://www.example.com/sitemap-relative-url.xml", - ] - - def test_alternate_url_locs(self): - sitemap = b""" - - - http://www.example.com/english/ - - - - - - """ - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/english/" - ] - - spider.sitemap_alternate_links = True - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/english/", - "http://www.example.com/deutsch/", - "http://www.example.com/schweiz-deutsch/", - "http://www.example.com/italiano/", - ] - - def test_sitemap_filter(self): - sitemap = b""" - - - http://www.example.com/english/ - 2010-01-01 - - - http://www.example.com/portuguese/ - 2005-01-01 - - """ - - class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, entries): - for entry in entries: - date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") - if date_time.year > 2008: - yield entry - - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/english/", - "http://www.example.com/portuguese/", - ] - - spider = FilteredSitemapSpider("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/english/" - ] - - def test_sitemap_filter_with_alternate_links(self): - sitemap = b""" - - - http://www.example.com/english/article_1/ - 2010-01-01 - - - - http://www.example.com/english/article_2/ - 2015-01-01 - - """ - - class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, entries): - for entry in entries: - alternate_links = entry.get("alternate", ()) - for link in alternate_links: - if "/deutsch/" in link: - entry["loc"] = link - yield entry - - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/english/article_1/", - "http://www.example.com/english/article_2/", - ] - - spider = FilteredSitemapSpider("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/deutsch/article_1/" - ] - - def test_sitemapindex_filter(self): - sitemap = b""" - - - http://www.example.com/sitemap1.xml - 2004-01-01T20:00:00+00:00 - - - http://www.example.com/sitemap2.xml - 2005-01-01 - - """ - - class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, entries): - for entry in entries: - date_time = datetime.strptime( - entry["lastmod"].split("T")[0], "%Y-%m-%d" - ) - if date_time.year > 2004: - yield entry - - r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) - spider = self.spider_class("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/sitemap1.xml", - "http://www.example.com/sitemap2.xml", - ] - - spider = FilteredSitemapSpider("example.com") - assert [req.url for req in spider._parse_sitemap(r)] == [ - "http://www.example.com/sitemap2.xml" - ] - - def test_compression_bomb_setting(self): - settings = {"DOWNLOAD_MAXSIZE": 10_000_000} - crawler = get_crawler(settings_dict=settings) - spider = self.spider_class.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request(url="https://example.com") - response = Response(url="https://example.com", body=body, request=request) - assert spider._get_sitemap_body(response) is None - - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_compression_bomb_spider_attr(self): - class DownloadMaxSizeSpider(self.spider_class): - download_maxsize = 10_000_000 - - crawler = get_crawler() - spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request(url="https://example.com") - response = Response(url="https://example.com", body=body, request=request) - assert spider._get_sitemap_body(response) is None - - def test_compression_bomb_request_meta(self): - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request( - url="https://example.com", meta={"download_maxsize": 10_000_000} - ) - response = Response(url="https://example.com", body=body, request=request) - assert spider._get_sitemap_body(response) is None - - def test_download_warnsize_setting(self): - settings = {"DOWNLOAD_WARNSIZE": 10_000_000} - crawler = get_crawler(settings_dict=settings) - spider = self.spider_class.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request(url="https://example.com") - response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: - spider._get_sitemap_body(response) - log.check( - ( - "scrapy.spiders.sitemap", - "WARNING", - ( - "<200 https://example.com> body size after decompression " - "(11511612 B) is larger than the download warning size " - "(10000000 B)." - ), - ), - ) - - @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") - def test_download_warnsize_spider_attr(self): - class DownloadWarnSizeSpider(self.spider_class): - download_warnsize = 10_000_000 - - crawler = get_crawler() - spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request( - url="https://example.com", meta={"download_warnsize": 10_000_000} - ) - response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: - spider._get_sitemap_body(response) - log.check( - ( - "scrapy.spiders.sitemap", - "WARNING", - ( - "<200 https://example.com> body size after decompression " - "(11511612 B) is larger than the download warning size " - "(10000000 B)." - ), - ), - ) - - def test_download_warnsize_request_meta(self): - crawler = get_crawler() - spider = self.spider_class.from_crawler(crawler, "example.com") - body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") - body = body_path.read_bytes() - request = Request( - url="https://example.com", meta={"download_warnsize": 10_000_000} - ) - response = Response(url="https://example.com", body=body, request=request) - with LogCapture( - "scrapy.spiders.sitemap", propagate=False, level=WARNING - ) as log: - spider._get_sitemap_body(response) - log.check( - ( - "scrapy.spiders.sitemap", - "WARNING", - ( - "<200 https://example.com> body size after decompression " - "(11511612 B) is larger than the download warning size " - "(10000000 B)." - ), - ), - ) - - @coroutine_test - async def test_sitemap_urls(self): - class TestSpider(self.spider_class): - name = "test" - sitemap_urls = ["https://toscrape.com/sitemap.xml"] - - crawler = get_crawler(TestSpider) - spider = TestSpider.from_crawler(crawler) - with warnings.catch_warnings(): - warnings.simplefilter("error") - requests = [request async for request in spider.start()] - - assert len(requests) == 1 - request = requests[0] - assert request.url == "https://toscrape.com/sitemap.xml" - assert request.dont_filter is False - assert request.callback == spider._parse_sitemap - - -class TestDeprecation: - def test_crawl_spider(self): - assert issubclass(CrawlSpider, Spider) - assert isinstance(CrawlSpider(name="foo"), Spider) - - class TestNoParseMethodSpider: spider_class = Spider diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py new file mode 100644 index 000000000..63010e195 --- /dev/null +++ b/tests/test_spider_crawl.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import re +import warnings + +import pytest +from w3lib.url import safe_url_string + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.http import HtmlResponse, Request, TextResponse +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule, Spider +from scrapy.utils.test import get_crawler +from tests.test_spider import TestSpider + + +class TestCrawlSpider(TestSpider): + test_body = b"""Page title + +

Item 12

+
+

About us

+
+ + """ + spider_class = CrawlSpider + + def test_rule_without_link_extractor(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(),) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + def test_process_links(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) + + def dummy_process_links(self, links): + return links + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + def test_process_links_filter(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="filter_process_links"),) + _test_regex = re.compile("nofollow") + + def filter_process_links(self, links): + return [link for link in links if not self._test_regex.search(link.url)] + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 2 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + ] + + def test_process_links_generator(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) + + def dummy_process_links(self, links): + yield from links + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + + def test_process_request(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + def process_request_change_domain(request, response): + return request.replace(url=request.url.replace(".org", ".com")) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = ( + Rule(LinkExtractor(), process_request=process_request_change_domain), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.com/somepage/item/12.html", + "http://example.com/about.html", + "http://example.com/nofollow.html", + ] + + def test_process_request_with_response(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + def process_request_meta_response_class(request, response): + request.meta["response_class"] = response.__class__.__name__ + return request + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = ( + Rule( + LinkExtractor(), process_request=process_request_meta_response_class + ), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] + + def test_process_request_instance_method(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = (Rule(LinkExtractor(), process_request="process_request_upper"),) + + def process_request_upper(self, request, response): + return request.replace(url=request.url.upper()) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"), + safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"), + safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"), + ] + + def test_process_request_instance_method_with_response(self): + response = HtmlResponse( + "http://example.org/somepage/index.html", body=self.test_body + ) + + class _CrawlSpider(self.spider_class): + name = "test" + allowed_domains = ["example.org"] + rules = ( + Rule( + LinkExtractor(), + process_request="process_request_meta_response_class", + ), + ) + + def process_request_meta_response_class(self, request, response): + request.meta["response_class"] = response.__class__.__name__ + return request + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + assert len(output) == 3 + assert all(isinstance(r, Request) for r in output) + assert [r.url for r in output] == [ + "http://example.org/somepage/item/12.html", + "http://example.org/about.html", + "http://example.org/nofollow.html", + ] + assert [r.meta["response_class"] for r in output] == [ + "HtmlResponse", + "HtmlResponse", + "HtmlResponse", + ] + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_follow_links_attribute_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + assert hasattr(spider, "_follow_links") + assert spider._follow_links + + settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False} + crawler = get_crawler(settings_dict=settings_dict) + spider = self.spider_class.from_crawler(crawler, "example.com") + assert hasattr(spider, "_follow_links") + assert not spider._follow_links + + def test_parse_response_use(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) + spider = _CrawlSpider() + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CrawlSpider\._parse_response method is deprecated", + ): + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + + def test_parse_response_override(self): + class _CrawlSpider(CrawlSpider): + def _parse_response(self, response, callback, cb_kwargs, follow=True): + pass + + name = "test" + start_urls = "https://www.example.com" + _follow_links = False + + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CrawlSpider\._parse_response method, which the", + ): + spider = _CrawlSpider() + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) + spider._parse_response( + TextResponse(spider.start_urls, body=b""), None, None + ) + + def test_parse_with_rules(self): + class _CrawlSpider(CrawlSpider): + name = "test" + start_urls = "https://www.example.com" + + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) + spider = _CrawlSpider() + spider.parse_with_rules( + TextResponse(spider.start_urls, body=b""), None, None + ) + + +class TestDeprecation: + def test_crawl_spider(self): + assert issubclass(CrawlSpider, Spider) + assert isinstance(CrawlSpider(name="foo"), Spider) diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py new file mode 100644 index 000000000..0af99ab6d --- /dev/null +++ b/tests/test_spider_sitemap.py @@ -0,0 +1,437 @@ +from __future__ import annotations + +import gzip +import re +from datetime import datetime +from io import BytesIO +from logging import WARNING +from pathlib import Path + +import pytest +from testfixtures import LogCapture + +from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse +from scrapy.spiders import SitemapSpider +from scrapy.utils.test import get_crawler +from tests import tests_datadir +from tests.test_spider import TestSpider +from tests.utils.decorators import coroutine_test + + +class TestSitemapSpider(TestSpider): + spider_class = SitemapSpider + + BODY = b"SITEMAP" + f = BytesIO() + g = gzip.GzipFile(fileobj=f, mode="w+b") + g.write(BODY) + g.close() + GZBODY = f.getvalue() + + def assertSitemapBody(self, response: Response, body: bytes | None) -> None: + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + assert spider._get_sitemap_body(response) == body + + def test_get_sitemap_body(self): + r = XmlResponse(url="http://www.example.com/", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + + r = HtmlResponse(url="http://www.example.com/", body=self.BODY) + self.assertSitemapBody(r, None) + + r = Response(url="http://www.example.com/favicon.ico", body=self.BODY) + self.assertSitemapBody(r, None) + + r = XmlResponse(url="http://www.example.com/", body=b"") + self.assertSitemapBody(r, b"") + + def test_get_sitemap_body_gzip_headers(self): + r = Response( + url="http://www.example.com/sitemap", + body=self.GZBODY, + headers={"content-type": "application/gzip"}, + request=Request("http://www.example.com/sitemap"), + ) + self.assertSitemapBody(r, self.BODY) + + def test_get_sitemap_body_xml_url(self): + r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + + def test_get_sitemap_body_xml_url_compressed(self): + r = Response( + url="http://www.example.com/sitemap.xml.gz", + body=self.GZBODY, + request=Request("http://www.example.com/sitemap"), + ) + self.assertSitemapBody(r, self.BODY) + + # .xml.gz but body decoded by HttpCompression middleware already + r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + + def test_get_sitemap_urls_from_robotstxt(self): + robots = b"""# Sitemap files +Sitemap: http://example.com/sitemap.xml +Sitemap: http://example.com/sitemap-product-index.xml +Sitemap: HTTP://example.com/sitemap-uppercase.xml +Sitemap: /sitemap-relative-url.xml +""" + + r = TextResponse(url="http://www.example.com/robots.txt", body=robots) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://www.example.com/sitemap-relative-url.xml", + ] + + def test_get_sitemap_urls_from_robotstxt_skips_invalid_utf8_urls(self): + robots = ( + b"User-agent: *\n" + b"Sitemap: http://example.com/\xff.xml\n" + b"Sitemap: http://example.com/ok.xml\n" + ) + + r = TextResponse(url="http://www.example.com/robots.txt", body=robots) + spider = self.spider_class("example.com") + + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://example.com/ok.xml", + ] + + def test_alternate_url_locs(self): + sitemap = b""" + + + http://www.example.com/english/ + + + + + + """ + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] + + spider.sitemap_alternate_links = True + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/italiano/", + ] + + def test_sitemap_filter(self): + sitemap = b""" + + + http://www.example.com/english/ + 2010-01-01 + + + http://www.example.com/portuguese/ + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d") + if date_time.year > 2008: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/", + "http://www.example.com/portuguese/", + ] + + spider = FilteredSitemapSpider("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/" + ] + + def test_sitemap_filter_with_alternate_links(self): + sitemap = b""" + + + http://www.example.com/english/article_1/ + 2010-01-01 + + + + http://www.example.com/english/article_2/ + 2015-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + alternate_links = entry.get("alternate", ()) + for link in alternate_links: + if "/deutsch/" in link: + entry["loc"] = link + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/english/article_1/", + "http://www.example.com/english/article_2/", + ] + + spider = FilteredSitemapSpider("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/deutsch/article_1/" + ] + + def test_sitemapindex_filter(self): + sitemap = b""" + + + http://www.example.com/sitemap1.xml + 2004-01-01T20:00:00+00:00 + + + http://www.example.com/sitemap2.xml + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + date_time = datetime.strptime( + entry["lastmod"].split("T")[0], "%Y-%m-%d" + ) + if date_time.year > 2004: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap1.xml", + "http://www.example.com/sitemap2.xml", + ] + + spider = FilteredSitemapSpider("example.com") + assert [req.url for req in spider._parse_sitemap(r)] == [ + "http://www.example.com/sitemap2.xml" + ] + + @pytest.mark.parametrize( + ("rule", "result"), + [(r"english", ["http://www.example.com/english/"]), (r"nonexistent", [])], + ) + def test_sitemap_filter_with_rule(self, rule: str, result: list[str]): + sitemap = b""" + + http://www.example.com/english/ + http://www.example.com/portuguese/ + """ + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + + class _RuleSpider(self.spider_class): # type: ignore[name-defined,misc] + sitemap_rules = [(rule, "parse")] + + spider = _RuleSpider("example.com") + urls = [req.url for req in spider._parse_sitemap(r)] + assert urls == result + + def test_parse_sitemap_empty_body(self): + r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"") + spider = self.spider_class("example.com") + + with LogCapture() as lc: + results = list(spider._parse_sitemap(r)) + + assert not results + + lc.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + "Ignoring invalid sitemap: <200 http://www.example.com/sitemap.xml>", + ) + ) + + def test_parse_sitemap_not_sitemap(self): + body = b""" + + sometext + sometext2 + """ + r = XmlResponse(url="http://www.example.com/random.xml", body=body) + spider = self.spider_class("example.com") + + results = list(spider._parse_sitemap(r)) + + assert not results + + @pytest.mark.parametrize( + ("follow", "result"), + [ + (r"1.xml", ["http://www.example.com/sitemap1.xml"]), + (re.compile(r"sitemap\d"), ["http://www.example.com/sitemap1.xml"]), + (r"nonexistent", []), + ], + ) + def test_sitemap_follow(self, follow, result): + sitemap = b""" + + + http://www.example.com/sitemap1.xml + + """ + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + + class _FollowSpider(self.spider_class): + sitemap_follow = [follow] + + spider = _FollowSpider("example.com") + urls = [req.url for req in spider._parse_sitemap(r)] + assert urls == result + + def test_compression_bomb_setting(self): + settings = {"DOWNLOAD_MAXSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + assert spider._get_sitemap_body(response) is None + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_compression_bomb_spider_attr(self): + class DownloadMaxSizeSpider(self.spider_class): + download_maxsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadMaxSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + assert spider._get_sitemap_body(response) is None + + def test_compression_bomb_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_maxsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + assert spider._get_sitemap_body(response) is None + + def test_download_warnsize_setting(self): + settings = {"DOWNLOAD_WARNSIZE": 10_000_000} + crawler = get_crawler(settings_dict=settings) + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request(url="https://example.com") + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_download_warnsize_spider_attr(self): + class DownloadWarnSizeSpider(self.spider_class): + download_warnsize = 10_000_000 + + crawler = get_crawler() + spider = DownloadWarnSizeSpider.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + def test_download_warnsize_request_meta(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, "example.com") + body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin") + body = body_path.read_bytes() + request = Request( + url="https://example.com", meta={"download_warnsize": 10_000_000} + ) + response = Response(url="https://example.com", body=body, request=request) + with LogCapture( + "scrapy.spiders.sitemap", propagate=False, level=WARNING + ) as log: + spider._get_sitemap_body(response) + log.check( + ( + "scrapy.spiders.sitemap", + "WARNING", + ( + "<200 https://example.com> body size after decompression " + "(11511612 B) is larger than the download warning size " + "(10000000 B)." + ), + ), + ) + + @coroutine_test + async def test_sitemap_urls(self): + class TestSpider(self.spider_class): + name = "test" + sitemap_urls = ["https://toscrape.com/sitemap.xml"] + + crawler = get_crawler(TestSpider) + spider = TestSpider.from_crawler(crawler) + requests = [request async for request in spider.start()] + + assert len(requests) == 1 + request = requests[0] + assert request.url == "https://toscrape.com/sitemap.xml" + assert request.dont_filter is False + assert request.callback == spider._parse_sitemap diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index 4e359fd33..9257e0232 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -1,15 +1,11 @@ from __future__ import annotations -import re -import warnings from asyncio import sleep from typing import Any import pytest -from testfixtures import LogCapture from scrapy import Spider, signals -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler @@ -48,9 +44,7 @@ class TestMain: async def parse(self, response): yield ITEM_A - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) + await self._test_spider(TestSpider, [ITEM_A]) @coroutine_test async def test_start(self): @@ -60,9 +54,7 @@ class TestMain: async def start(self): yield ITEM_A - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) + await self._test_spider(TestSpider, [ITEM_A]) @coroutine_test async def test_start_subclass(self): @@ -73,84 +65,7 @@ class TestMain: class TestSpider(BaseSpider): name = "test" - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) - - @coroutine_test - async def test_deprecated(self): - class TestSpider(Spider): - name = "test" - - def start_requests(self): - yield ITEM_A - - with pytest.warns(ScrapyDeprecationWarning): - await self._test_spider(TestSpider, [ITEM_A]) - - @coroutine_test - async def test_deprecated_subclass(self): - class BaseSpider(Spider): - def start_requests(self): - yield ITEM_A - - class TestSpider(BaseSpider): - name = "test" - - # The warning must be about the base class and not the subclass. - with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"): - await self._test_spider(TestSpider, [ITEM_A]) - - @coroutine_test - async def test_universal(self): - class TestSpider(Spider): - name = "test" - - async def start(self): - yield ITEM_A - - def start_requests(self): - yield ITEM_B - - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) - - @coroutine_test - async def test_universal_subclass(self): - class BaseSpider(Spider): - async def start(self): - yield ITEM_A - - def start_requests(self): - yield ITEM_B - - class TestSpider(BaseSpider): - name = "test" - - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_spider(TestSpider, [ITEM_A]) - - @coroutine_test - async def test_start_deprecated_super(self): - class TestSpider(Spider): - name = "test" - - async def start(self): - for item_or_request in super().start_requests(): - yield item_or_request - - msg = "use Spider.start() instead" - with pytest.warns(ScrapyDeprecationWarning, match=re.escape(msg)) as ws: - await self._test_spider(TestSpider, []) - - for w in ws: - if isinstance(w.message, ScrapyDeprecationWarning) and msg in str( - w.message - ): - assert w.filename.endswith("test_spider_start.py") - break + await self._test_spider(TestSpider, [ITEM_A]) async def _test_start(self, start_, expected_items=None): class TestSpider(Spider): @@ -176,24 +91,3 @@ class TestMain: yield ITEM_A await self._test_start(start, [ITEM_A]) - - # Exceptions - - @coroutine_test - async def test_deprecated_non_generator_exception(self): - class TestSpider(Spider): - name = "test" - - def start_requests(self): - raise RuntimeError - - with ( - LogCapture() as log, - pytest.warns( - ScrapyDeprecationWarning, - match=r"defines the deprecated start_requests\(\) method", - ), - ): - await self._test_spider(TestSpider, []) - - assert "in start_requests\n raise RuntimeError" in str(log) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 245507c0b..d85942f90 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,7 +1,6 @@ import contextlib import shutil import sys -import warnings from pathlib import Path from unittest import mock @@ -137,50 +136,38 @@ class TestSpiderLoader: SpiderLoader.from_settings(settings) def test_bad_spider_modules_warning(self): - with warnings.catch_warnings(record=True) as w: - module = "tests.test_spiderloader.test_spiders.doesnotexist" - settings = Settings( - {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} - ) + module = "tests.test_spiderloader.test_spiders.doesnotexist" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + with pytest.warns(RuntimeWarning, match="Could not load spiders from module"): spider_loader = SpiderLoader.from_settings(settings) - if str(w[0].message).startswith("_SixMetaPathImporter"): - # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, - # at least until all six versions we can import (including botocore.vendored.six) - # are updated to 1.16.0+ - w.pop(0) - assert "Could not load spiders from module" in str(w[0].message) - spiders = spider_loader.list() - assert not spiders + spiders = spider_loader.list() + assert not spiders def test_syntax_error_exception(self): module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings({"SPIDER_MODULES": [module]}) with mock.patch.object(SpiderLoader, "_load_spiders") as m: m.side_effect = SyntaxError - settings = Settings({"SPIDER_MODULES": [module]}) with pytest.raises(SyntaxError): SpiderLoader.from_settings(settings) def test_syntax_error_warning(self): - with ( - warnings.catch_warnings(record=True) as w, - mock.patch.object(SpiderLoader, "_load_spiders") as m, - ): + module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + with mock.patch.object(SpiderLoader, "_load_spiders") as m: m.side_effect = SyntaxError - module = "tests.test_spiderloader.test_spiders.spider1" - settings = Settings( - {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} - ) - spider_loader = SpiderLoader.from_settings(settings) - if str(w[0].message).startswith("_SixMetaPathImporter"): - # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, - # at least until all six versions we can import (including botocore.vendored.six) - # are updated to 1.16.0+ - w.pop(0) - assert "Could not load spiders from module" in str(w[0].message) + with pytest.warns( + RuntimeWarning, match="Could not load spiders from module" + ): + spider_loader = SpiderLoader.from_settings(settings) - spiders = spider_loader.list() - assert not spiders + spiders = spider_loader.list() + assert not spiders class TestDuplicateSpiderNameLoader: @@ -190,21 +177,17 @@ class TestDuplicateSpiderNameLoader: # copy 1 spider module so as to have duplicate spider name shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py") - with warnings.catch_warnings(record=True) as w: + msg = r"""There are several spiders with the same name: + + Spider3 named 'spider3' \(in test_spiders_xxx\.spider3\) + + Spider3 named 'spider3' \(in test_spiders_xxx\.spider3dupe\) + + This can cause unexpected behavior\.""" + with pytest.warns(UserWarning, match=msg): spider_loader = SpiderLoader.from_settings(settings) - - assert len(w) == 1 - msg = str(w[0].message) - assert "several spiders with the same name" in msg - assert "'spider3'" in msg - assert msg.count("'spider3'") == 2 - - assert "'spider1'" not in msg - assert "'spider2'" not in msg - assert "'spider4'" not in msg - - spiders = set(spider_loader.list()) - assert spiders == {"spider1", "spider2", "spider3", "spider4"} + spiders = set(spider_loader.list()) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} def test_multiple_dupename_warning(self, spider_loader_env): settings, spiders_dir = spider_loader_env @@ -213,23 +196,21 @@ class TestDuplicateSpiderNameLoader: shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py") shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py") - with warnings.catch_warnings(record=True) as w: + msg = r"""There are several spiders with the same name: + + Spider1 named 'spider1' \(in test_spiders_xxx\.spider1\) + + Spider1 named 'spider1' \(in test_spiders_xxx\.spider1dupe\) + + Spider2 named 'spider2' \(in test_spiders_xxx\.spider2\) + + Spider2 named 'spider2' \(in test_spiders_xxx\.spider2dupe\) + + This can cause unexpected behavior\.""" + with pytest.warns(UserWarning, match=msg): spider_loader = SpiderLoader.from_settings(settings) - - assert len(w) == 1 - msg = str(w[0].message) - assert "several spiders with the same name" in msg - assert "'spider1'" in msg - assert msg.count("'spider1'") == 2 - - assert "'spider2'" in msg - assert msg.count("'spider2'") == 2 - - assert "'spider3'" not in msg - assert "'spider4'" not in msg - - spiders = set(spider_loader.list()) - assert spiders == {"spider1", "spider2", "spider3", "spider4"} + spiders = set(spider_loader.list()) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} class CustomSpiderLoader(SpiderLoader): @@ -252,3 +233,4 @@ def test_dummy_spider_loader(spider_loader_env): assert not spider_loader.list() with pytest.raises(KeyError): spider_loader.load("spider1") + assert not spider_loader.find_by_request(Request("http://example.com")) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 9651e3f8b..e5891474b 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -2,11 +2,10 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable from inspect import isasyncgen -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from unittest import mock import pytest -from testfixtures import LogCapture from twisted.internet import defer from scrapy.core.spidermw import SpiderMiddlewareManager @@ -27,7 +26,7 @@ if TYPE_CHECKING: class TestSpiderMiddleware: - def setup_method(self): + def setup_method(self) -> None: self.request = Request("http://example.com/index.html") self.response = Response(self.request.url, request=self.request) self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}}) @@ -39,11 +38,10 @@ class TestSpiderMiddleware: Raise exception in case of failure. """ - def scrape_func( + async def scrape_func( response: Response | Failure, request: Request - ) -> defer.Deferred[Iterable[Any]]: - it = mock.MagicMock() - return defer.succeed(it) + ) -> Iterable[Any]: + return mock.MagicMock() return await self.mwman.scrape_response_async( scrape_func, self.response, self.request @@ -64,20 +62,6 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): await self._scrape_response() -class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): - """Invalid return value for process_spider_output method""" - - @coroutine_test - async def test_invalid_process_spider_output(self): - class InvalidProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result): - return 1 - - self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) - with pytest.raises(_InvalidOutput): - await self._scrape_response() - - class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" @@ -88,13 +72,15 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): return 1 class RaiseExceptionProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result): + async def process_spider_output(self, response, result): raise RuntimeError + yield # pylint: disable=unreachable self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) + it = await self._scrape_response() with pytest.raises(_InvalidOutput): - await self._scrape_response() + await collect_asyncgen(it) class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): @@ -107,13 +93,15 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): return None class RaiseExceptionProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result): + async def process_spider_output(self, response, result): 1 / 0 + yield self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) + it = await self._scrape_response() with pytest.raises(ZeroDivisionError): - await self._scrape_response() + await collect_asyncgen(it) class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): @@ -122,7 +110,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): Should work for process_spider_output and, when it's supported, process_start. """ - ITEM_TYPE: type | tuple + ITEM_TYPE: type | tuple[type, ...] RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod @@ -141,7 +129,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): async def _scrape_func( self, response: Response | Failure, request: Request ) -> Iterable[Any] | AsyncIterator[Any]: - return self._callback() + return cast("Iterable[Any] | AsyncIterator[Any]", self._callback()) async def _get_middleware_result( self, *mw_classes: type[Any], start_index: int | None = None @@ -156,43 +144,19 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): self._scrape_func, self.response, self.request ) - async def _test_simple_base( - self, - *mw_classes: type[Any], - downgrade: bool = False, - start_index: int | None = None, - ) -> None: - with LogCapture() as log: - result = await self._get_middleware_result( - *mw_classes, start_index=start_index - ) - assert isinstance(result, Iterable) - result_list = list(result) - assert len(result_list) == self.RESULT_COUNT - assert isinstance(result_list[0], self.ITEM_TYPE) - assert ("downgraded to a non-async" in str(log)) == downgrade - assert ("doesn't support asynchronous spider output" in str(log)) == ( - ProcessSpiderOutputSimpleMiddleware in mw_classes - ) - async def _test_asyncgen_base( self, *mw_classes: type[Any], - downgrade: bool = False, start_index: int | None = None, ) -> None: - with LogCapture() as log: - result = await self._get_middleware_result( - *mw_classes, start_index=start_index - ) + result = await self._get_middleware_result(*mw_classes, start_index=start_index) assert isinstance(result, AsyncIterator) result_list = await collect_asyncgen(result) assert len(result_list) == self.RESULT_COUNT assert isinstance(result_list[0], self.ITEM_TYPE) - assert ("downgraded to a non-async" in str(log)) == downgrade -class ProcessSpiderOutputSimpleMiddleware: +class ProcessSpiderOutputSyncMiddleware: def process_spider_output(self, response, result): yield from result @@ -233,53 +197,36 @@ class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware): """process_spider_output tests for simple callbacks""" ITEM_TYPE = dict - MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware + MW_SYNC = ProcessSpiderOutputSyncMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware @coroutine_test - async def test_simple(self): - """Simple mw""" - await self._test_simple_base(self.MW_SIMPLE) + async def test_sync(self): + """Unsupported sync mw""" + with pytest.raises( + TypeError, match=r"doesn't support asynchronous spider output" + ): + await self._get_middleware_result(self.MW_SYNC) @coroutine_test async def test_asyncgen(self): - """Asyncgen mw; upgrade""" + """Asyncgen mw""" await self._test_asyncgen_base(self.MW_ASYNCGEN) - @coroutine_test - async def test_simple_asyncgen(self): - """Simple mw -> asyncgen mw; upgrade""" - await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) - - @coroutine_test - async def test_asyncgen_simple(self): - """Asyncgen mw -> simple mw; upgrade then downgrade""" - await self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True) - @coroutine_test async def test_universal(self): """Universal mw""" - await self._test_simple_base(self.MW_UNIVERSAL) - - @coroutine_test - async def test_universal_simple(self): - """Universal mw -> simple mw""" - await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL) - - @coroutine_test - async def test_simple_universal(self): - """Simple mw -> universal mw""" - await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE) + await self._test_asyncgen_base(self.MW_UNIVERSAL) @coroutine_test async def test_universal_asyncgen(self): - """Universal mw -> asyncgen mw; upgrade""" + """Universal mw -> asyncgen mw""" await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) @coroutine_test async def test_asyncgen_universal(self): - """Asyncgen mw -> universal mw; upgrade""" + """Asyncgen mw -> universal mw""" await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) @@ -290,59 +237,6 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): for item in super()._callback(): yield item - @coroutine_test - async def test_simple(self): - """Simple mw; downgrade""" - await self._test_simple_base(self.MW_SIMPLE, downgrade=True) - - @coroutine_test - async def test_simple_asyncgen(self): - """Simple mw -> asyncgen mw; downgrade then upgrade""" - await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True) - - @coroutine_test - async def test_universal(self): - """Universal mw""" - await self._test_asyncgen_base(self.MW_UNIVERSAL) - - @coroutine_test - async def test_universal_simple(self): - """Universal mw -> simple mw; downgrade""" - await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True) - - @coroutine_test - async def test_simple_universal(self): - """Simple mw -> universal mw; downgrade""" - await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True) - - -class ProcessSpiderOutputNonIterableMiddleware: - def process_spider_output(self, response, result): - return - - -class ProcessSpiderOutputCoroutineMiddleware: - async def process_spider_output(self, response, result): - return result - - -class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): - @coroutine_test - async def test_non_iterable(self): - with pytest.raises( - _InvalidOutput, - match=r"\.process_spider_output must return an iterable, got ", - ): - await self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware) - - @coroutine_test - async def test_coroutine(self): - with pytest.raises( - _InvalidOutput, - match=r"\.process_spider_output must be an asynchronous generator", - ): - await self._get_middleware_result(ProcessSpiderOutputCoroutineMiddleware) - class ProcessStartSimpleMiddleware: async def process_start(self, start): @@ -416,11 +310,11 @@ class TestUniversalMiddlewareManager: return SpiderMiddlewareManager.from_crawler(crawler) def test_simple_mw(self, mwman: SpiderMiddlewareManager) -> None: - mw = ProcessSpiderOutputSimpleMiddleware() - mwman._add_middleware(mw) - assert ( - mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable - ) + mw = ProcessSpiderOutputSyncMiddleware() + with pytest.raises( + TypeError, match=r"doesn't support asynchronous spider output" + ): + mwman._add_middleware(mw) def test_async_mw(self, mwman: SpiderMiddlewareManager) -> None: mw = ProcessSpiderOutputAsyncGenMiddleware() @@ -432,9 +326,8 @@ class TestUniversalMiddlewareManager: def test_universal_mw(self, mwman: SpiderMiddlewareManager) -> None: mw = ProcessSpiderOutputUniversalMiddleware() mwman._add_middleware(mw) - assert mwman.methods["process_spider_output"][0] == ( - mw.process_spider_output, - mw.process_spider_output_async, + assert ( + mwman.methods["process_spider_output"][0] == mw.process_spider_output_async # pylint: disable=comparison-with-callable ) def test_universal_mw_no_sync( @@ -442,7 +335,7 @@ class TestUniversalMiddlewareManager: ) -> None: mwman._add_middleware(UniversalMiddlewareNoSync()) assert ( - "UniversalMiddlewareNoSync has process_spider_output_async" + "UniversalMiddlewareNoSync has process_spider_output_async()" " without process_spider_output" in caplog.text ) assert mwman.methods["process_spider_output"][0] is None @@ -466,7 +359,7 @@ class TestUniversalMiddlewareManager: mwman._add_middleware(UniversalMiddlewareBothAsync()) assert ( "UniversalMiddlewareBothAsync.process_spider_output " - "is an async generator function while process_spider_output_async exists" + "is an async generator function while process_spider_output_async() exists" in caplog.text ) assert mwman.methods["process_spider_output"][0] is None @@ -474,7 +367,6 @@ class TestUniversalMiddlewareManager: class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict - MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware @@ -489,66 +381,22 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): self._scrape_func, self.response, self.request ) - @coroutine_test - async def test_just_builtin(self): - await self._test_simple_base() - - @coroutine_test - async def test_builtin_simple(self): - await self._test_simple_base(self.MW_SIMPLE, start_index=1000) - - @coroutine_test - async def test_builtin_async(self): - """Upgrade""" - await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) - - @coroutine_test - async def test_builtin_universal(self): - await self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) - - @coroutine_test - async def test_simple_builtin(self): - await self._test_simple_base(self.MW_SIMPLE) - - @coroutine_test - async def test_async_builtin(self): - """Upgrade""" - await self._test_asyncgen_base(self.MW_ASYNCGEN) - - @coroutine_test - async def test_universal_builtin(self): - await self._test_simple_base(self.MW_UNIVERSAL) - - -class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): - async def _callback(self) -> Any: - for item in super()._callback(): - yield item - @coroutine_test async def test_just_builtin(self): await self._test_asyncgen_base() - @coroutine_test - async def test_builtin_simple(self): - """Downgrade""" - await self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) - @coroutine_test async def test_builtin_async(self): + """Upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) @coroutine_test async def test_builtin_universal(self): await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) - @coroutine_test - async def test_simple_builtin(self): - """Downgrade""" - await self._test_simple_base(self.MW_SIMPLE, downgrade=True) - @coroutine_test async def test_async_builtin(self): + """Upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN) @coroutine_test @@ -556,39 +404,31 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): await self._test_asyncgen_base(self.MW_UNIVERSAL) +class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): + async def _callback(self) -> Any: + for item in super()._callback(): + yield item + + class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict - MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware - MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware def _callback(self) -> Any: 1 / 0 - async def _test_asyncgen_nodowngrade(self, *mw_classes: type[Any]) -> None: - with pytest.raises( - _InvalidOutput, - match=r"Async iterable returned from .+ cannot be downgraded", - ): - await self._get_middleware_result(*mw_classes) - @coroutine_test async def test_exc_simple(self): """Simple exc mw""" - await self._test_simple_base(self.MW_EXC_SIMPLE) + await self._test_asyncgen_base(self.MW_EXC_SIMPLE) @coroutine_test async def test_exc_async(self): """Async exc mw""" await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) - @coroutine_test - async def test_exc_simple_simple(self): - """Simple exc mw -> simple output mw""" - await self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE) - @coroutine_test async def test_exc_async_async(self): """Async exc mw -> async output mw""" @@ -599,25 +439,27 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): """Simple exc mw -> async output mw; upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) - @coroutine_test - async def test_exc_async_simple(self): - """Async exc mw -> simple output mw; cannot work as downgrading is not supported""" - await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) - class TestDeprecatedSpiderArg(TestSpiderMiddleware): @coroutine_test async def test_deprecated_mw_spider_arg(self): - class DeprecatedSpiderArgMiddleware: + class DeprecatedSpiderArgMiddleware1: def process_spider_input(self, response, spider): return None - def process_spider_output(self, response, result, spider): + async def process_spider_output(self, response, result, spider): 1 / 0 + yield + class DeprecatedSpiderArgMiddleware2: def process_spider_exception(self, response, exception, spider): return [] + with pytest.warns( + ScrapyDeprecationWarning, + match=r"process_spider_exception\(\) requires a spider argument", + ): + self.mwman._add_middleware(DeprecatedSpiderArgMiddleware2()) with ( pytest.warns( ScrapyDeprecationWarning, @@ -627,13 +469,10 @@ class TestDeprecatedSpiderArg(TestSpiderMiddleware): ScrapyDeprecationWarning, match=r"process_spider_output\(\) requires a spider argument", ), - pytest.warns( - ScrapyDeprecationWarning, - match=r"process_spider_exception\(\) requires a spider argument", - ), ): - self.mwman._add_middleware(DeprecatedSpiderArgMiddleware()) - await self._scrape_response() + self.mwman._add_middleware(DeprecatedSpiderArgMiddleware1()) + it = await self._scrape_response() + await collect_asyncgen(it) @coroutine_test async def test_deprecated_mwman_spider_arg(self): diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py index da570904d..70326f3f1 100644 --- a/tests/test_spidermiddleware_base.py +++ b/tests/test_spidermiddleware_base.py @@ -7,7 +7,9 @@ import pytest from scrapy import Request, Spider from scrapy.http import Response from scrapy.spidermiddlewares.base import BaseSpiderMiddleware +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -18,7 +20,8 @@ def crawler() -> Crawler: return get_crawler(Spider) -def test_trivial(crawler: Crawler) -> None: +@coroutine_test +async def test_trivial(crawler: Crawler) -> None: class TrivialSpiderMiddleware(BaseSpiderMiddleware): pass @@ -29,12 +32,13 @@ def test_trivial(crawler: Crawler) -> None: spider_output = [test_req, {"foo": "bar"}] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), - list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert processed == [test_req, {"foo": "bar"}] -def test_processed_request(crawler: Crawler) -> None: +@coroutine_test +async def test_processed_request(crawler: Crawler) -> None: class ProcessReqSpiderMiddleware(BaseSpiderMiddleware): def get_processed_request( self, request: Request, response: Response | None @@ -52,7 +56,7 @@ def test_processed_request(crawler: Crawler) -> None: spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), - list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert len(processed) == 3 assert isinstance(processed[0], Request) @@ -62,7 +66,8 @@ def test_processed_request(crawler: Crawler) -> None: assert processed[2].url == "data:30," -def test_processed_item(crawler: Crawler) -> None: +@coroutine_test +async def test_processed_item(crawler: Crawler) -> None: class ProcessItemSpiderMiddleware(BaseSpiderMiddleware): def get_processed_item(self, item: Any, response: Response | None) -> Any: if item["foo"] == 2: @@ -76,12 +81,13 @@ def test_processed_item(crawler: Crawler) -> None: spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), - list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert processed == [{"foo": 1}, test_req, {"foo": 30}] -def test_processed_both(crawler: Crawler) -> None: +@coroutine_test +async def test_processed_both(crawler: Crawler) -> None: class ProcessBothSpiderMiddleware(BaseSpiderMiddleware): def get_processed_request( self, request: Request, response: Response | None @@ -113,7 +119,7 @@ def test_processed_both(crawler: Crawler) -> None: ] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), - list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] + await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert len(processed) == 4 assert isinstance(processed[0], Request) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index b15f68dd0..6a054ded5 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,17 +1,19 @@ from __future__ import annotations import logging +from typing import TYPE_CHECKING import pytest -from testfixtures import LogCapture from scrapy.http import Request, Response from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer class _HttpErrorSpider(MockServerSpider): @@ -88,7 +90,7 @@ class TestHttpErrorMiddleware: def test_process_spider_exception( self, mw: HttpErrorMiddleware, res404: Response ) -> None: - assert mw.process_spider_exception(res404, HttpError(res404)) == [] + assert mw.process_spider_exception(res404, HttpError(res404)) == () assert mw.process_spider_exception(res404, Exception()) is None def test_handle_httpstatus_list( @@ -192,65 +194,66 @@ class TestHttpErrorMiddlewareHandleAll: class TestHttpErrorMiddlewareIntegrational: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - @inline_callbacks_test - def test_middleware_works(self): + @coroutine_test + async def test_middleware_works(self, mockserver: MockServer) -> None: crawler = get_crawler(_HttpErrorSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) assert not crawler.spider.skipped assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} + assert crawler.stats get_value = crawler.stats.get_value assert get_value("httperror/response_ignored_count") == 3 assert get_value("httperror/response_ignored_status_count/404") == 1 assert get_value("httperror/response_ignored_status_count/402") == 1 assert get_value("httperror/response_ignored_status_count/500") == 1 - @inline_callbacks_test - def test_logging(self): + @coroutine_test + async def test_logging( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(_HttpErrorSpider) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver, bypass_status_codes={402}) + assert isinstance(crawler.spider, _HttpErrorSpider) assert crawler.spider.parsed == {"200", "402"} assert crawler.spider.skipped == {"402"} assert crawler.spider.failed == {"404", "500"} - assert "Ignoring response <404" in str(log) - assert "Ignoring response <500" in str(log) - assert "Ignoring response <200" not in str(log) - assert "Ignoring response <402" not in str(log) + assert "Ignoring response <404" in caplog.text + assert "Ignoring response <500" in caplog.text + assert "Ignoring response <200" not in caplog.text + assert "Ignoring response <402" not in caplog.text - @inline_callbacks_test - def test_logging_level(self): + @coroutine_test + async def test_logging_level( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # HttpError logs ignored responses with level INFO crawler = get_crawler(_HttpErrorSpider) - with LogCapture(level=logging.INFO) as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} - assert "Ignoring response <402" in str(log) - assert "Ignoring response <404" in str(log) - assert "Ignoring response <500" in str(log) - assert "Ignoring response <200" not in str(log) + assert "Ignoring response <402" in caplog.text + assert "Ignoring response <404" in caplog.text + assert "Ignoring response <500" in caplog.text + assert "Ignoring response <200" not in caplog.text # with level WARNING, we shouldn't capture anything from HttpError + caplog.clear() crawler = get_crawler(_HttpErrorSpider) - with LogCapture(level=logging.WARNING) as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.WARNING): + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} - assert "Ignoring response <402" not in str(log) - assert "Ignoring response <404" not in str(log) - assert "Ignoring response <500" not in str(log) - assert "Ignoring response <200" not in str(log) + assert "Ignoring response <402" not in caplog.text + assert "Ignoring response <404" not in caplog.text + assert "Ignoring response <500" not in caplog.text + assert "Ignoring response <200" not in caplog.text diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 8cf08dd94..afdfaa322 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -169,8 +169,8 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS # ================================================================================ # (4) exceptions from a middleware process_spider_output method (generator) class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - for r in result: + async def process_spider_output(self, response, result): + async for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r @@ -182,8 +182,8 @@ class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware): class GeneratorFailMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - for r in result: + async def process_spider_output(self, response, result): + async for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r raise LookupError @@ -201,8 +201,8 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): class GeneratorRecoverMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - for r in result: + async def process_spider_output(self, response, result): + async for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r @@ -237,86 +237,6 @@ class GeneratorOutputChainSpider(Spider): yield {"processed": ["parse-second-item"]} -# ================================================================================ -# (5) exceptions from a middleware process_spider_output method (not generator) - - -class _NotGeneratorDoNothingMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - out = [] - for r in result: - r["processed"].append(f"{self.__class__.__name__}.process_spider_output") - out.append(r) - return out - - def process_spider_exception(self, response, exception): - method = f"{self.__class__.__name__}.process_spider_exception" - self.crawler.spider.logger.info( - "%s: %s caught", method, exception.__class__.__name__ - ) - - -class NotGeneratorFailMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - out = [] - for r in result: - r["processed"].append(f"{self.__class__.__name__}.process_spider_output") - out.append(r) - raise ReferenceError - - def process_spider_exception(self, response, exception): - method = f"{self.__class__.__name__}.process_spider_exception" - self.crawler.spider.logger.info( - "%s: %s caught", method, exception.__class__.__name__ - ) - return [{"processed": [method]}] - - -class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddleware): - pass - - -class NotGeneratorRecoverMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - out = [] - for r in result: - r["processed"].append(f"{self.__class__.__name__}.process_spider_output") - out.append(r) - return out - - def process_spider_exception(self, response, exception): - method = f"{self.__class__.__name__}.process_spider_exception" - self.crawler.spider.logger.info( - "%s: %s caught", method, exception.__class__.__name__ - ) - return [{"processed": [method]}] - - -class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): - pass - - -class NotGeneratorOutputChainSpider(Spider): - name = "NotGeneratorOutputChainSpider" - custom_settings = { - "SPIDER_MIDDLEWARES": { - NotGeneratorFailMiddleware: 10, - NotGeneratorDoNothingAfterFailureMiddleware: 8, - NotGeneratorRecoverMiddleware: 5, - NotGeneratorDoNothingAfterRecoveryMiddleware: 3, - }, - } - - async def start(self): - yield Request(self.mockserver.url("/status?n=200")) - - def parse(self, response): - return [ - {"processed": ["parse-first-item"]}, - {"processed": ["parse-second-item"]}, - ] - - # ================================================================================ class TestSpiderMiddleware: mockserver: MockServer @@ -481,41 +401,3 @@ class TestSpiderMiddleware: assert str(item_from_callback) in str(log4) assert str(item_recovered) in str(log4) assert "parse-second-item" not in str(log4) - - @coroutine_test - async def test_not_a_generator_output_chain(self): - """ - (5) An exception from a middleware's process_spider_output method should be sent - to the process_spider_exception method from the next middleware in the chain. - The result of the recovery by the process_spider_exception method should be handled - by the process_spider_output method from the next middleware. - The final item count should be 1 (from the process_spider_exception chain, the items - from the spider callback are lost) - """ - log5 = await self.crawl_log(NotGeneratorOutputChainSpider) - assert "'item_scraped_count': 1" in str(log5) - assert ( - "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught" - in str(log5) - ) - assert ( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught" - in str(log5) - ) - assert ( - "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught" - not in str(log5) - ) - assert ( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught" - not in str(log5) - ) - item_recovered = { - "processed": [ - "NotGeneratorRecoverMiddleware.process_spider_exception", - "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", - ] - } - assert str(item_recovered) in str(log5) - assert "parse-first-item" not in str(log5) - assert "parse-second-item" not in str(log5) diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index bccfc4432..67ce3a920 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -1,10 +1,8 @@ -import warnings from asyncio import sleep import pytest from scrapy import Spider, signals -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.test_spider_start import SLEEP_SECONDS @@ -15,7 +13,6 @@ from .utils.decorators import coroutine_test ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} ITEM_C = {"id": "c"} -ITEM_D = {"id": "d"} class AsyncioSleepSpiderMiddleware: @@ -38,15 +35,6 @@ class TwistedSleepSpiderMiddleware: yield item_or_request -class UniversalSpiderMiddleware: - async def process_start(self, start): - async for item_or_request in start: - yield item_or_request - - def process_start_requests(self, start_requests, spider): - raise NotImplementedError - - # Spiders and spider middlewares for TestMain._test_wrap @@ -57,27 +45,6 @@ class ModernWrapSpider(Spider): yield ITEM_B -class ModernWrapSpiderSubclass(ModernWrapSpider): - name = "test" - - -class UniversalWrapSpider(Spider): - name = "test" - - async def start(self): - yield ITEM_B - - def start_requests(self): - yield ITEM_D - - -class DeprecatedWrapSpider(Spider): - name = "test" - - def start_requests(self): - yield ITEM_B - - class ModernWrapSpiderMiddleware: async def process_start(self, start): yield ITEM_A @@ -86,26 +53,6 @@ class ModernWrapSpiderMiddleware: yield ITEM_C -class UniversalWrapSpiderMiddleware: - async def process_start(self, start): - yield ITEM_A - async for item_or_request in start: - yield item_or_request - yield ITEM_C - - def process_start_requests(self, start, spider): - yield ITEM_A - yield from start - yield ITEM_C - - -class DeprecatedWrapSpiderMiddleware: - def process_start_requests(self, start, spider): - yield ITEM_A - yield from start - yield ITEM_C - - class TestMain: async def _test(self, spider_middlewares, spider_cls, expected_items): actual_items = [] @@ -126,199 +73,9 @@ class TestMain: expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C] await self._test([spider_middleware], spider_cls, expected_items) - async def _test_douple_wrap(self, smw1, smw2, spider_cls, expected_items=None): - expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C] - await self._test([smw1, smw2], spider_cls, expected_items) - @coroutine_test async def test_modern_mw_modern_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) - - @coroutine_test - async def test_modern_mw_universal_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider) - - @coroutine_test - async def test_modern_mw_deprecated_spider(self): - with pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" - ): - await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider) - - @coroutine_test - async def test_universal_mw_modern_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider) - - @coroutine_test - async def test_universal_mw_universal_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider) - - @coroutine_test - async def test_universal_mw_deprecated_spider(self): - with pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" - ): - await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider) - - @coroutine_test - async def test_deprecated_mw_modern_spider(self): - with ( - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ), - pytest.raises( - ValueError, match=r"only compatible with \(deprecated\) spiders" - ), - ): - await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider) - - @coroutine_test - async def test_deprecated_mw_modern_spider_subclass(self): - with ( - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ), - pytest.raises( - ValueError, - match=r"^\S+?\.ModernWrapSpider \(inherited by \S+?.ModernWrapSpiderSubclass\) .*? only compatible with \(deprecated\) spiders", - ), - ): - await self._test_wrap( - DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass - ) - - @coroutine_test - async def test_deprecated_mw_universal_spider(self): - with pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ): - await self._test_wrap( - DeprecatedWrapSpiderMiddleware, - UniversalWrapSpider, - [ITEM_A, ITEM_D, ITEM_C], - ) - - @coroutine_test - async def test_deprecated_mw_deprecated_spider(self): - with ( - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ), - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" - ), - ): - await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider) - - @coroutine_test - async def test_modern_mw_universal_mw_modern_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - UniversalWrapSpiderMiddleware, - ModernWrapSpider, - ) - - @coroutine_test - async def test_modern_mw_deprecated_mw_modern_spider(self): - with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - ModernWrapSpider, - ) - - @coroutine_test - async def test_universal_mw_deprecated_mw_modern_spider(self): - with ( - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ), - pytest.raises( - ValueError, match=r"only compatible with \(deprecated\) spiders" - ), - ): - await self._test_douple_wrap( - UniversalWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - ModernWrapSpider, - ) - - @coroutine_test - async def test_modern_mw_universal_mw_universal_spider(self): - with warnings.catch_warnings(): - warnings.simplefilter("error") - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - UniversalWrapSpiderMiddleware, - UniversalWrapSpider, - ) - - @coroutine_test - async def test_modern_mw_deprecated_mw_universal_spider(self): - with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - UniversalWrapSpider, - ) - - @coroutine_test - async def test_universal_mw_deprecated_mw_universal_spider(self): - with pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ): - await self._test_douple_wrap( - UniversalWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - UniversalWrapSpider, - [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C], - ) - - @coroutine_test - async def test_modern_mw_universal_mw_deprecated_spider(self): - with pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" - ): - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - UniversalWrapSpiderMiddleware, - DeprecatedWrapSpider, - ) - - @coroutine_test - async def test_modern_mw_deprecated_mw_deprecated_spider(self): - with pytest.raises(ValueError, match=r"trying to combine spider middlewares"): - await self._test_douple_wrap( - ModernWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - DeprecatedWrapSpider, - ) - - @coroutine_test - async def test_universal_mw_deprecated_mw_deprecated_spider(self): - with ( - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)" - ), - pytest.warns( - ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)" - ), - ): - await self._test_douple_wrap( - UniversalWrapSpiderMiddleware, - DeprecatedWrapSpiderMiddleware, - DeprecatedWrapSpider, - ) + await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider) async def _test_sleep(self, spider_middlewares): class TestSpider(Spider): @@ -341,12 +98,12 @@ class TestMain: [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware] ) - @pytest.mark.requires_reactor + @pytest.mark.requires_reactor # needs a reactor for twisted_sleep() @coroutine_test async def test_twisted_sleep_single(self): await self._test_sleep([TwistedSleepSpiderMiddleware]) - @pytest.mark.requires_reactor + @pytest.mark.requires_reactor # needs a reactor for twisted_sleep() @coroutine_test async def test_twisted_sleep_multiple(self): await self._test_sleep( diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 68fcad98c..b3a434c3f 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,12 +1,12 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, cast +from typing import Any from urllib.parse import urlparse import pytest -from scrapy.downloadermiddlewares.redirect import RedirectMiddleware +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spidermiddlewares.referer import ( @@ -31,13 +31,9 @@ from scrapy.spidermiddlewares.referer import ( StrictOriginWhenCrossOriginPolicy, UnsafeUrlPolicy, ) -from scrapy.utils.spider import DefaultSpider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler - -if TYPE_CHECKING: - from collections.abc import Sequence - - from scrapy.crawler import Crawler +from tests.utils.decorators import coroutine_test class TestRefererMiddleware: @@ -841,19 +837,20 @@ class TestRequestMetaSettingFallback: request_meta, policy_class, check_warning, - ) in self.params[3:]: + ) in self.params: mw = RefererMiddleware(Settings(settings)) response = Response(origin, headers=response_headers) request = Request(target, meta=request_meta) - with warnings.catch_warnings(record=True) as w: + if check_warning: + with pytest.warns( + RuntimeWarning, match="Could not load referrer policy" + ): + policy = mw.policy(response, request) + else: policy = mw.policy(response, request) - assert isinstance(policy, policy_class) - - if check_warning: - assert len(w) == 1 - assert w[0].category is RuntimeWarning, w[0].message + assert isinstance(policy, policy_class) class TestSettingsPolicyByName: @@ -970,391 +967,151 @@ class TestPolicyHeaderPrecedence004( resp_headers = {"Referrer-Policy": ""} -class TestReferrerOnRedirect(TestRefererMiddleware): - settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} - scenarii: Sequence[ - tuple[str, str, tuple[tuple[int, str], ...], bytes | None, bytes | None] - ] = [ # type: ignore[assignment] - ( - "http://scrapytest.org/1", # parent - "http://scrapytest.org/2", # target - ( - # redirections: code, URL - (301, "http://scrapytest.org/3"), - (301, "http://scrapytest.org/4"), - ), - b"http://scrapytest.org/1", # expected initial referer - b"http://scrapytest.org/1", # expected referer for the redirection request - ), - ( - "https://scrapytest.org/1", - "https://scrapytest.org/2", - ( - # redirecting to non-secure URL - (301, "http://scrapytest.org/3"), - ), - b"https://scrapytest.org/1", - b"https://scrapytest.org/1", - ), - ( - "https://scrapytest.org/1", - "https://scrapytest.com/2", - ( - # redirecting to non-secure URL: different origin - (301, "http://scrapytest.com/3"), - ), - b"https://scrapytest.org/1", - b"https://scrapytest.org/1", - ), +class TestPolicyMethodResponseParamRename: + def setup_method(self): + self.crawler = get_crawler() + self.mw = build_from_crawler(RefererMiddleware, self.crawler) + self.request = Request("http://www.example.com") + self.response = Response("http://www.example.com") + + def test_pos_string(self): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a response URL to RefererMiddleware\.policy\(\)", + ): + self.mw.policy("http://old.com", self.request) + + def test_pos_response(self): + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", + category=ScrapyDeprecationWarning, + message=r"Passing 'resp_or_url' is deprecated", + ) + self.mw.policy(self.response, self.request) + + def test_key_resp_or_url(self): + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing 'resp_or_url' is deprecated" + ): + self.mw.policy(resp_or_url=self.response, request=self.request) + + def test_key_response(self): + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", + category=ScrapyDeprecationWarning, + message=r"Passing 'resp_or_url' is deprecated", + ) + self.mw.policy(response=self.response, request=self.request) + + def test_key_response_string(self): + with pytest.warns(ScrapyDeprecationWarning, match="Passing a response URL"): + self.mw.policy(response="http://old.com", request=self.request) + + def test_both_resp_or_url_and_response(self): + with pytest.raises( + TypeError, match="Cannot pass both 'response' and 'resp_or_url'" + ): + self.mw.policy( + response=self.response, resp_or_url=self.response, request=self.request + ) + + +@coroutine_test +async def test_response_policy_only_supports_policy_names(): + crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"}) + mw = build_from_crawler(RefererMiddleware, crawler) + + async def input_result(): + yield Request("https://example.com/") + + response = Response( + "https://example.com/", + headers={ + "Referrer-Policy": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy" + }, + ) + with pytest.warns( + RuntimeWarning, + match=r"Could not load referrer policy 'scrapy\.spidermiddlewares\.referer\.NoReferrerWhenDowngradePolicy' \(import paths from the response Referrer-Policy header are not allowed\)", + ): + output = [ + request + async for request in mw.process_spider_output_async( + response, input_result() + ) + ] + assert len(output) == 1 + assert b"Referer" not in output[0].headers + + response = Response( + "https://example.com/", + headers={"Referrer-Policy": "no-referrer-when-downgrade"}, + ) + output = [ + request + async for request in mw.process_spider_output_async(response, input_result()) ] - - @pytest.fixture - def crawler(self) -> Crawler: - crawler = get_crawler(DefaultSpider, self.settings) - crawler.spider = crawler._create_spider() - return crawler - - @pytest.fixture - def referrermw(self, crawler: Crawler) -> RefererMiddleware: - return RefererMiddleware.from_crawler(crawler) - - @pytest.fixture - def redirectmw(self, crawler: Crawler) -> RedirectMiddleware: - return RedirectMiddleware.from_crawler(crawler) - - def test( # type: ignore[override] - self, - crawler: Crawler, - referrermw: RefererMiddleware, - redirectmw: RedirectMiddleware, - ) -> None: - for ( - parent, - target, - redirections, - init_referrer, - final_referrer, - ) in self.scenarii: - response = self.get_response(parent) - request = self.get_request(target) - - out = list(referrermw.process_spider_output(response, [request])) - assert out[0].headers.get("Referer") == init_referrer - - for status, url in redirections: - response = Response( - request.url, headers={"Location": url}, status=status - ) - request = cast( - "Request", redirectmw.process_response(request, response) - ) - assert crawler.spider - referrermw.request_scheduled(request, crawler.spider) - - assert isinstance(request, Request) - assert request.headers.get("Referer") == final_referrer + assert len(output) == 1 + assert output[0].headers == {b"Referer": [b"https://example.com/"]} -class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): - """ - No Referrer policy never sets the "Referer" header. - HTTP redirections should not change that. - """ +@coroutine_test +async def test_referer_policies_setting(): + crawler = get_crawler( + settings_dict={ + "REFERRER_POLICY": "no-referrer", + "REFERRER_POLICIES": { + "no-referrer-when-downgrade": None, + "custom-policy": CustomPythonOrgPolicy, + "": CustomPythonOrgPolicy, + }, + } + ) + mw = build_from_crawler(RefererMiddleware, crawler) - settings = {"REFERRER_POLICY": "no-referrer"} - scenarii = [ - ( - "http://scrapytest.org/1", # parent - "http://scrapytest.org/2", # target - ( - # redirections: code, URL - (301, "http://scrapytest.org/3"), - (301, "http://scrapytest.org/4"), - ), - None, # expected initial "Referer" - None, # expected "Referer" for the redirection request - ), - ( - "https://scrapytest.org/1", - "https://scrapytest.org/2", - ((301, "http://scrapytest.org/3"),), - None, - None, - ), - ( - "https://scrapytest.org/1", - "https://example.com/2", # different origin - ((301, "http://scrapytest.com/3"),), - None, - None, - ), + async def input_result(): + yield Request("https://example.com/") + + # "no-referrer-when-downgrade": None, + response = Response( + "https://example.com/", + headers={"Referrer-Policy": "no-referrer-when-downgrade"}, + ) + with pytest.warns( + RuntimeWarning, + match=r"Could not load referrer policy 'no-referrer-when-downgrade'", + ): + output = [ + request + async for request in mw.process_spider_output_async( + response, input_result() + ) + ] + assert len(output) == 1 + assert b"Referer" not in output[0].headers + + # "custom-policy": CustomPythonOrgPolicy, + response = Response( + "https://example.com/", + headers={"Referrer-Policy": "custom-policy"}, + ) + output = [ + request + async for request in mw.process_spider_output_async(response, input_result()) ] + assert len(output) == 1 + assert output[0].headers == {b"Referer": [b"https://python.org/"]} - -class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): - """ - Same Origin policy sends the full URL as "Referer" if the target origin - is the same as the parent response (same protocol, same domain, same port). - - HTTP redirections to a different domain or a lower secure level - should have the "Referer" removed. - """ - - settings = {"REFERRER_POLICY": "same-origin"} - scenarii = [ - ( - "http://scrapytest.org/101", # origin - "http://scrapytest.org/102", # target - ( - # redirections: code, URL - (301, "http://scrapytest.org/103"), - (301, "http://scrapytest.org/104"), - ), - b"http://scrapytest.org/101", # expected initial "Referer" - b"http://scrapytest.org/101", # expected referer for the redirection request - ), - ( - "https://scrapytest.org/201", - "https://scrapytest.org/202", - ( - # redirecting from secure to non-secure URL == different origin - (301, "http://scrapytest.org/203"), - ), - b"https://scrapytest.org/201", - None, - ), - ( - "https://scrapytest.org/301", - "https://scrapytest.org/302", - ( - # different domain == different origin - (301, "http://example.com/303"), - ), - b"https://scrapytest.org/301", - None, - ), - ] - - -class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect): - """ - Strict Origin policy will always send the "origin" as referrer - (think of it as the parent URL without the path part), - unless the security level is lower and no "Referer" is sent. - - Redirections from secure to non-secure URLs should have the - "Referrer" header removed if necessary. - """ - - settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN} - scenarii = [ - ( - "http://scrapytest.org/101", - "http://scrapytest.org/102", - ( - (301, "http://scrapytest.org/103"), - (301, "http://scrapytest.org/104"), - ), - b"http://scrapytest.org/", # send origin - b"http://scrapytest.org/", # redirects to same origin: send origin - ), - ( - "https://scrapytest.org/201", - "https://scrapytest.org/202", - ( - # redirecting to non-secure URL: no referrer - (301, "http://scrapytest.org/203"), - ), - b"https://scrapytest.org/", - None, - ), - ( - "https://scrapytest.org/301", - "https://scrapytest.org/302", - ( - # redirecting to non-secure URL (different domain): no referrer - (301, "http://example.com/303"), - ), - b"https://scrapytest.org/", - None, - ), - ( - "http://scrapy.org/401", - "http://example.com/402", - ((301, "http://scrapytest.org/403"),), - b"http://scrapy.org/", - b"http://scrapy.org/", - ), - ( - "https://scrapy.org/501", - "https://example.com/502", - ( - # HTTPS all along, so origin referrer is kept as-is - (301, "https://google.com/503"), - (301, "https://facebook.com/504"), - ), - b"https://scrapy.org/", - b"https://scrapy.org/", - ), - ( - "https://scrapytest.org/601", - "http://scrapytest.org/602", # TLS to non-TLS: no referrer - ( - ( - 301, - "https://scrapytest.org/603", - ), # TLS URL again: (still) no referrer - ), - None, - None, - ), - ] - - -class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): - """ - Origin When Cross-Origin policy sends the full URL as "Referer", - unless the target's origin is different (different domain, different protocol) - in which case only the origin is sent. - - Redirections to a different origin should strip the "Referer" - to the parent origin. - """ - - settings = {"REFERRER_POLICY": POLICY_ORIGIN_WHEN_CROSS_ORIGIN} - scenarii = [ - ( - "http://scrapytest.org/101", # origin - "http://scrapytest.org/102", # target + redirection - ( - # redirections: code, URL - (301, "http://scrapytest.org/103"), - (301, "http://scrapytest.org/104"), - ), - b"http://scrapytest.org/101", # expected initial referer - b"http://scrapytest.org/101", # expected referer for the redirection request - ), - ( - "https://scrapytest.org/201", - "https://scrapytest.org/202", - ( - # redirecting to non-secure URL: send origin - (301, "http://scrapytest.org/203"), - ), - b"https://scrapytest.org/201", - b"https://scrapytest.org/", - ), - ( - "https://scrapytest.org/301", - "https://scrapytest.org/302", - ( - # redirecting to non-secure URL (different domain): send origin - (301, "http://example.com/303"), - ), - b"https://scrapytest.org/301", - b"https://scrapytest.org/", - ), - ( - "http://scrapy.org/401", - "http://example.com/402", - ((301, "http://scrapytest.org/403"),), - b"http://scrapy.org/", - b"http://scrapy.org/", - ), - ( - "https://scrapy.org/501", - "https://example.com/502", - ( - # all different domains: send origin - (301, "https://google.com/503"), - (301, "https://facebook.com/504"), - ), - b"https://scrapy.org/", - b"https://scrapy.org/", - ), - ( - "https://scrapytest.org/301", - "http://scrapytest.org/302", # TLS to non-TLS: send origin - ((301, "https://scrapytest.org/303"),), # TLS URL again: send origin (also) - b"https://scrapytest.org/", - b"https://scrapytest.org/", - ), - ] - - -class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): - """ - Strict Origin When Cross-Origin policy sends the full URL as "Referer", - unless the target's origin is different (different domain, different protocol) - in which case only the origin is sent... - Unless there's also a downgrade in security and then the "Referer" header - is not sent. - - Redirections to a different origin should strip the "Referer" to the parent origin, - and from https:// to http:// will remove the "Referer" header. - """ - - settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} - scenarii = [ - ( - "http://scrapytest.org/101", # origin - "http://scrapytest.org/102", # target + redirection - ( - # redirections: code, URL - (301, "http://scrapytest.org/103"), - (301, "http://scrapytest.org/104"), - ), - b"http://scrapytest.org/101", # expected initial referer - b"http://scrapytest.org/101", # expected referer for the redirection request - ), - ( - "https://scrapytest.org/201", - "https://scrapytest.org/202", - ( - # redirecting to non-secure URL: do not send the "Referer" header - (301, "http://scrapytest.org/203"), - ), - b"https://scrapytest.org/201", - None, - ), - ( - "https://scrapytest.org/301", - "https://scrapytest.org/302", - ( - # redirecting to non-secure URL (different domain): send origin - (301, "http://example.com/303"), - ), - b"https://scrapytest.org/301", - None, - ), - ( - "http://scrapy.org/401", - "http://example.com/402", - ((301, "http://scrapytest.org/403"),), - b"http://scrapy.org/", - b"http://scrapy.org/", - ), - ( - "https://scrapy.org/501", - "https://example.com/502", - ( - # all different domains: send origin - (301, "https://google.com/503"), - (301, "https://facebook.com/504"), - ), - b"https://scrapy.org/", - b"https://scrapy.org/", - ), - ( - "https://scrapytest.org/601", - "http://scrapytest.org/602", # TLS to non-TLS: do not send "Referer" - ( - ( - 301, - "https://scrapytest.org/603", - ), # TLS URL again: (still) send nothing - ), - None, - None, - ), + # "": CustomPythonOrgPolicy, + response = Response( + "https://example.com/", + headers={"Referrer-Policy": ""}, + ) + output = [ + request + async for request in mw.process_spider_output_async(response, input_result()) ] + assert len(output) == 1 + assert output[0].headers == {b"Referer": [b"https://python.org/"]} diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py index 76976d962..26397d37c 100644 --- a/tests/test_spidermiddleware_start.py +++ b/tests/test_spidermiddleware_start.py @@ -23,20 +23,3 @@ class TestMiddleware: async for request in mw.process_start(start()) ] assert result == [True, True, False, "foo"] - - @coroutine_test - async def test_sync(self): - crawler = get_crawler(Spider) - mw = build_from_crawler(StartSpiderMiddleware, crawler) - - def start(): - yield Request("data:,1") - yield Request("data:,2", meta={"is_start_request": True}) - yield Request("data:,2", meta={"is_start_request": False}) - yield Request("data:,2", meta={"is_start_request": "foo"}) - - result = [ - request.meta["is_start_request"] - for request in mw.process_start_requests(start(), Spider("test")) - ] - assert result == [True, True, False, "foo"] diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 491fc88f7..e44cfca90 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,4 +1,7 @@ +from __future__ import annotations + from datetime import datetime, timezone +from typing import TYPE_CHECKING import pytest @@ -7,8 +10,11 @@ from scrapy.extensions.spiderstate import SpiderState from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +if TYPE_CHECKING: + from pathlib import Path -def test_store_load(tmp_path): + +def test_store_load(tmp_path: Path) -> None: jobdir = str(tmp_path) spider = Spider(name="default") @@ -16,6 +22,7 @@ def test_store_load(tmp_path): ss = SpiderState(jobdir) ss.spider_opened(spider) + assert hasattr(spider, "state") spider.state["one"] = 1 spider.state["dt"] = dt ss.spider_closed(spider) @@ -23,21 +30,23 @@ def test_store_load(tmp_path): spider2 = Spider(name="default") ss2 = SpiderState(jobdir) ss2.spider_opened(spider2) - assert spider.state == {"one": 1, "dt": dt} + assert hasattr(spider2, "state") + assert spider2.state == {"one": 1, "dt": dt} ss2.spider_closed(spider2) -def test_state_attribute(): +def test_state_attribute() -> None: # state attribute must be present if jobdir is not set, to provide a # consistent interface spider = Spider(name="default") ss = SpiderState() ss.spider_opened(spider) + assert hasattr(spider, "state") assert spider.state == {} ss.spider_closed(spider) -def test_not_configured(): +def test_not_configured() -> None: crawler = get_crawler(Spider) with pytest.raises(NotConfigured): SpiderState.from_crawler(crawler) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 8283d3d5d..ddc12766c 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -120,7 +120,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as", ) as exc_info: q.push(lambda x: x) - if hasattr(sys, "pypy_version_info"): + if sys.version_info >= (3, 14) or hasattr(sys, "pypy_version_info"): assert isinstance(exc_info.value.__context__, pickle.PicklingError) else: assert isinstance(exc_info.value.__context__, AttributeError) diff --git a/tests/test_stats.py b/tests/test_stats.py index 2869d302e..6e6aa0cc4 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -29,6 +29,7 @@ def spider(crawler: Crawler) -> Spider: class TestCoreStatsExtension: + @mock.patch("scrapy.extensions.corestats.monotonic", return_value=0) @mock.patch("scrapy.extensions.corestats.datetime") def test_core_stats_default_stats_collector( self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider @@ -93,8 +94,13 @@ class TestStatsCollector: assert stats.get_value("test2") == 35 stats.min_value("test4", 7) assert stats.get_value("test4") == 7 + stats.set_stats({"replaced": "stats"}) + assert stats.get_stats() == {"replaced": "stats"} + stats.clear_stats() + assert stats.get_stats() == {} - def test_dummy_collector(self, crawler: Crawler) -> None: + def test_dummy_collector(self) -> None: + crawler = get_crawler(Spider, {"STATS_DUMP": False}) stats = DummyStatsCollector(crawler) assert stats.get_stats() == {} assert stats.get_value("anything") is None @@ -103,9 +109,11 @@ class TestStatsCollector: stats.inc_value("v1") stats.max_value("v2", 100) stats.min_value("v3", 100) + stats.set_stats({"key": "val"}) stats.open_spider() stats.set_value("test", "value") assert stats.get_stats() == {} + stats.close_spider() def test_deprecated_spider_arg(self, crawler: Crawler, spider: Spider) -> None: stats = StatsCollector(crawler) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 5489fd948..5532b4a31 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -21,10 +21,10 @@ if TYPE_CHECKING: class TestAsyncio: - @pytest.mark.requires_reactor # needs a reactor or an event loop for is_asyncio_available() - def test_is_asyncio_available(self, reactor_pytest: str) -> None: + @coroutine_test + async def test_is_asyncio_available(self, reactor_pytest: str) -> None: # the result should depend only on the pytest --reactor argument - assert is_asyncio_available() == (reactor_pytest == "asyncio") + assert is_asyncio_available() == (reactor_pytest != "default") @pytest.mark.only_asyncio @@ -83,6 +83,7 @@ class TestParallelAsyncio: max_parallel_count, ) assert list(range(length)) == sorted(results) + assert parallel_count[0] == 0 assert max_parallel_count[0] <= self.CONCURRENT_ITEMS @coroutine_test @@ -101,13 +102,14 @@ class TestParallelAsyncio: max_parallel_count, ) assert list(range(length)) == sorted(results) + assert parallel_count[0] == 0 assert max_parallel_count[0] <= self.CONCURRENT_ITEMS -@pytest.mark.requires_reactor # needs a running event loop for AsyncioLoopingCall.start() @pytest.mark.only_asyncio class TestAsyncioLoopingCall: - def test_looping_call(self): + @coroutine_test + async def test_looping_call(self): func = mock.MagicMock() looping_call = AsyncioLoopingCall(func) looping_call.start(1, now=False) @@ -116,21 +118,24 @@ class TestAsyncioLoopingCall: assert not looping_call.running assert not func.called - def test_looping_call_now(self): + @coroutine_test + async def test_looping_call_now(self): func = mock.MagicMock() looping_call = AsyncioLoopingCall(func) looping_call.start(1) looping_call.stop() assert func.called - def test_looping_call_already_running(self): + @coroutine_test + async def test_looping_call_already_running(self): looping_call = AsyncioLoopingCall(lambda: None) looping_call.start(1) with pytest.raises(RuntimeError): looping_call.start(1) looping_call.stop() - def test_looping_call_interval(self): + @coroutine_test + async def test_looping_call_interval(self): looping_call = AsyncioLoopingCall(lambda: None) with pytest.raises(ValueError, match="Interval must be greater than 0"): looping_call.start(0) @@ -138,7 +143,8 @@ class TestAsyncioLoopingCall: looping_call.start(-1) assert not looping_call.running - def test_looping_call_bad_function(self): + @coroutine_test + async def test_looping_call_bad_function(self): looping_call = AsyncioLoopingCall(Deferred) with pytest.raises(TypeError): looping_call.start(0.1) diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index b1532ca77..0627a4b93 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -213,10 +213,12 @@ class TestCurlToRequestKwargs: def test_ignore_unknown_options(self): # case 1: ignore_unknown_options=True: + curl_command = "curl --bar --baz http://www.example.com" + expected_result = {"method": "GET", "url": "http://www.example.com"} with warnings.catch_warnings(): # avoid warning when executing tests - warnings.simplefilter("ignore") - curl_command = "curl --bar --baz http://www.example.com" - expected_result = {"method": "GET", "url": "http://www.example.com"} + warnings.filterwarnings( + "ignore", category=UserWarning, message="Unrecognized options:" + ) assert curl_to_request_kwargs(curl_command) == expected_result # case 2: ignore_unknown_options=False (raise exception): diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index c58b02ca5..ba6b82503 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,7 +1,7 @@ import copy -import warnings from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping +from typing import Any import pytest @@ -20,7 +20,7 @@ from scrapy.utils.python import garbage_collect class TestCaseInsensitiveDictBase(ABC): @property @abstractmethod - def dict_class(self) -> type[MutableMapping]: + def dict_class(self) -> type[MutableMapping[str, Any]]: raise NotImplementedError def test_init_dict(self): @@ -206,7 +206,7 @@ class TestCaseInsensitiveDictBase(ABC): class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): - dict_class = CaseInsensitiveDict + dict_class = CaseInsensitiveDict # type: ignore[assignment] def test_repr(self): d1 = self.dict_class({"foo": "bar"}) @@ -226,18 +226,12 @@ class TestCaselessDict(TestCaseInsensitiveDictBase): dict_class = CaselessDict def test_deprecation_message(self): - with warnings.catch_warnings(record=True) as caught: - warnings.filterwarnings("always", category=ScrapyDeprecationWarning) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"scrapy.utils.datatypes.CaselessDict is deprecated", + ): self.dict_class({"foo": "bar"}) - assert len(caught) == 1 - assert issubclass(caught[0].category, ScrapyDeprecationWarning) - assert ( - str(caught[0].message) - == "scrapy.utils.datatypes.CaselessDict is deprecated," - " please use scrapy.utils.datatypes.CaseInsensitiveDict instead" - ) - class TestSequenceExclude: def test_list(self): diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py new file mode 100644 index 000000000..9743e1a50 --- /dev/null +++ b/tests/test_utils_decorators.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import warnings + +import pytest +from twisted.internet.defer import Deferred + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthread +from scrapy.utils.defer import maybe_deferred_to_future +from tests.utils.decorators import coroutine_test + + +class TestDeprecated: + def test_warns_and_still_calls(self): + @deprecated() + def add(a, b): + return a + b + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Call to deprecated function add\." + ): + result = add(2, 3) + + assert result == 5 + + def test_use_instead_in_message(self): + @deprecated(use_instead="other_function") + def old(): + return None + + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Call to deprecated function old\. Use other_function instead\.", + ): + old() + + def test_applied_without_parentheses(self): + @deprecated + def square(x): + return x * x + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Call to deprecated function square\." + ) as record: + result = square(4) + + assert result == 16 + # No "Use ... instead." part when applied directly to the function. + assert "instead" not in str(record[0].message) + + +class TestInthread: + @coroutine_test + async def test_returns_deferred_with_result(self): + @inthread + def multiply(a, b): + return a * b + + deferred = multiply(6, 7) + assert isinstance(deferred, Deferred) + assert await maybe_deferred_to_future(deferred) == 42 + + +class TestWarnSpiderArg: + def test_sync_warns_with_spider_arg(self): + @_warn_spider_arg + def parse(response, spider=None): + return response + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing a 'spider' argument" + ): + assert parse("response", spider="spider") == "response" + + def test_sync_no_warning_without_spider_arg(self): + @_warn_spider_arg + def parse(response, spider=None): + return response + + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) + assert parse("response") == "response" + + @coroutine_test + async def test_async_warns_with_spider_arg(self): + @_warn_spider_arg + async def parse(response, spider=None): + return response + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing a 'spider' argument" + ): + assert await parse("response", spider="spider") == "response" + + @coroutine_test + async def test_asyncgen_warns_with_spider_arg(self): + @_warn_spider_arg + async def parse(response, spider=None): + yield response + + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing a 'spider' argument" + ): + results = [item async for item in parse("response", spider="spider")] + + assert results == ["response"] diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 296bd6224..978c24f5a 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -6,7 +6,7 @@ from asyncio import Future from typing import TYPE_CHECKING, Any import pytest -from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import ( @@ -19,13 +19,13 @@ from scrapy.utils.defer import ( mustbe_deferred, parallel_async, ) -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Generator -@pytest.mark.requires_reactor +@pytest.mark.requires_reactor # mustbe_deferred() requires a reactor @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestMustbeDeferred: @inline_callbacks_test @@ -65,26 +65,6 @@ class TestMustbeDeferred: yield dfd -def cb1(value, arg1, arg2): - return f"(cb1 {value} {arg1} {arg2})" - - -def cb2(value, arg1, arg2): - return succeed(f"(cb2 {value} {arg1} {arg2})") - - -def cb3(value, arg1, arg2): - return f"(cb3 {value} {arg1} {arg2})" - - -def cb_fail(value, arg1, arg2): - raise TypeError - - -def eb1(failure, arg1, arg2): - return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" - - class TestIterErrback: def test_iter_errback_good(self): def itergood() -> Generator[int, None, None]: @@ -109,9 +89,8 @@ class TestIterErrback: assert isinstance(errors[0].value, ZeroDivisionError) -@pytest.mark.requires_reactor class TestAiterErrback: - @deferred_f_from_coro_f + @coroutine_test async def test_aiter_errback_good(self): async def itergood() -> AsyncGenerator[int, None]: for x in range(10): @@ -122,7 +101,7 @@ class TestAiterErrback: assert out == list(range(10)) assert not errors - @deferred_f_from_coro_f + @coroutine_test async def test_iter_errback_bad(self): async def iterbad() -> AsyncGenerator[int, None]: for x in range(10): @@ -137,23 +116,18 @@ class TestAiterErrback: assert isinstance(errors[0].value, ZeroDivisionError) -@pytest.mark.requires_reactor class TestAsyncDefTestsuite: - @deferred_f_from_coro_f - async def test_deferred_f_from_coro_f(self): + @coroutine_test + async def test_coroutine_test(self): pass - @deferred_f_from_coro_f - async def test_deferred_f_from_coro_f_generator(self): - yield - @pytest.mark.xfail(reason="Checks that the test is actually executed", strict=True) - @deferred_f_from_coro_f - async def test_deferred_f_from_coro_f_xfail(self): + @coroutine_test + async def test_coroutine_test_xfail(self): raise RuntimeError("This is expected to be raised") -@pytest.mark.requires_reactor +@pytest.mark.requires_reactor # parallel_async() requires a reactor class TestParallelAsync: """This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. @@ -347,9 +321,8 @@ class TestDeferredFFromCoroF: @pytest.mark.only_asyncio -@pytest.mark.requires_reactor class TestDeferredToFuture: - @deferred_f_from_coro_f + @coroutine_test async def test_deferred(self): d = Deferred() result = deferred_to_future(d) @@ -358,7 +331,7 @@ class TestDeferredToFuture: future_result = await result assert future_result == 42 - @deferred_f_from_coro_f + @coroutine_test async def test_wrapped_coroutine(self): async def c_f() -> int: return 42 @@ -369,7 +342,7 @@ class TestDeferredToFuture: future_result = await result assert future_result == 42 - @deferred_f_from_coro_f + @coroutine_test async def test_wrapped_coroutine_asyncio(self): async def c_f() -> int: await asyncio.sleep(0.01) @@ -383,11 +356,8 @@ class TestDeferredToFuture: @pytest.mark.only_asyncio -# needs a reactor or an event loop for is_asyncio_available() -# (for maybe_deferred_to_future()) -@pytest.mark.requires_reactor class TestMaybeDeferredToFutureAsyncio: - @deferred_f_from_coro_f + @coroutine_test async def test_deferred(self): d = Deferred() result = maybe_deferred_to_future(d) @@ -396,7 +366,7 @@ class TestMaybeDeferredToFutureAsyncio: future_result = await result assert future_result == 42 - @deferred_f_from_coro_f + @coroutine_test async def test_wrapped_coroutine(self): async def c_f() -> int: return 42 @@ -407,7 +377,7 @@ class TestMaybeDeferredToFutureAsyncio: future_result = await result assert future_result == 42 - @deferred_f_from_coro_f + @coroutine_test async def test_wrapped_coroutine_asyncio(self): async def c_f() -> int: await asyncio.sleep(0.01) @@ -421,11 +391,9 @@ class TestMaybeDeferredToFutureAsyncio: @pytest.mark.only_not_asyncio -# needs a reactor or an event loop for is_asyncio_available() -# (for maybe_deferred_to_future()) -@pytest.mark.requires_reactor class TestMaybeDeferredToFutureNotAsyncio: - def test_deferred(self): + @coroutine_test + async def test_deferred(self): d = Deferred() result = maybe_deferred_to_future(d) assert isinstance(result, Deferred) diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index c5425d99d..0706fec99 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,7 +1,6 @@ import inspect import warnings from unittest import mock -from warnings import WarningMessage import pytest @@ -22,36 +21,26 @@ class NewName(SomeBaseClass): class TestWarnWhenSubclassed: - def _mywarnings( - self, w: list[WarningMessage], category: type[Warning] = MyWarning - ) -> list[WarningMessage]: - return [x for x in w if x.category is MyWarning] - def test_no_warning_on_definition(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) create_deprecated_class("Deprecated", NewName) - w = self._mywarnings(w) - assert w == [] - def test_subclassing_warning_message(self): + msg = ( + r"tests\.test_utils_deprecate\.UserClass inherits from " + r"deprecated class tests\.test_utils_deprecate\.Deprecated, " + r"please inherit from tests\.test_utils_deprecate\.NewName." + r" \(warning only on first subclass, there may be others\)" + ) Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) - - with warnings.catch_warnings(record=True) as w: + with pytest.warns(MyWarning, match=msg) as w: class UserClass(Deprecated): pass - w = self._mywarnings(w) - assert len(w) == 1 - assert ( - str(w[0].message) == "tests.test_utils_deprecate.UserClass inherits from " - "deprecated class tests.test_utils_deprecate.Deprecated, " - "please inherit from tests.test_utils_deprecate.NewName." - " (warning only on first subclass, there may be others)" - ) assert w[0].lineno == inspect.getsourcelines(UserClass)[1] def test_custom_class_paths(self): @@ -63,62 +52,77 @@ class TestWarnWhenSubclassed: warn_category=MyWarning, ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass", + ): class UserClass(Deprecated): pass + with pytest.warns( + MyWarning, + match=r"bar\.OldClass is deprecated, instantiate foo\.NewClass instead", + ): _ = Deprecated() - w = self._mywarnings(w) - assert len(w) == 2 - assert "foo.NewClass" in str(w[0].message) - assert "bar.OldClass" in str(w[0].message) - assert "foo.NewClass" in str(w[1].message) - assert "bar.OldClass" in str(w[1].message) - def test_subclassing_warns_only_on_direct_children(self): Deprecated = create_deprecated_class( "Deprecated", NewName, warn_once=False, warn_category=MyWarning ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match="UserClass inherits from deprecated class", + ): class UserClass(Deprecated): pass + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) + class NoWarnOnMe(UserClass): pass - w = self._mywarnings(w) - assert len(w) == 1 - assert "UserClass" in str(w[0].message) - def test_subclassing_warns_once_by_default(self): Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match="UserClass inherits from deprecated class", + ): class UserClass(Deprecated): pass + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) + class FooClass(Deprecated): pass class BarClass(Deprecated): pass - w = self._mywarnings(w) - assert len(w) == 1 - assert "UserClass" in str(w[0].message) - def test_warning_on_instance(self): Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) + with pytest.warns(MyWarning) as w: + _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) + + w = [x for x in w if x.category is MyWarning] + assert len(w) == 1 + assert ( + str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " + "instantiate tests.test_utils_deprecate.NewName instead." + ) + assert w[0].lineno == lineno + # ignore subclassing warnings with warnings.catch_warnings(): warnings.simplefilter("ignore", MyWarning) @@ -126,29 +130,20 @@ class TestWarnWhenSubclassed: class UserClass(Deprecated): pass - with warnings.catch_warnings(record=True) as w: - _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) - _ = UserClass() # subclass instances don't warn - - w = self._mywarnings(w) - assert len(w) == 1 - assert ( - str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate tests.test_utils_deprecate.NewName instead." - ) - assert w[0].lineno == lineno + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) + UserClass() # subclass instances don't warn def test_warning_auto_message(self): - with warnings.catch_warnings(record=True) as w: - Deprecated = create_deprecated_class("Deprecated", NewName) + Deprecated = create_deprecated_class("Deprecated", NewName) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName", + ): class UserClass2(Deprecated): pass - msg = str(w[0].message) - assert "tests.test_utils_deprecate.NewName" in msg - assert "tests.test_utils_deprecate.Deprecated" in msg - def test_issubclass(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) @@ -224,8 +219,8 @@ class TestWarnWhenSubclassed: create_deprecated_class("Deprecated", New) def test_deprecate_subclass_of_deprecated_class(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) @@ -236,33 +231,26 @@ class TestWarnWhenSubclassed: warn_category=MyWarning, ) - w = self._mywarnings(w) - assert len(w) == 0, [str(warning) for warning in w] - - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match=r"AlsoDeprecated is deprecated, instantiate foo\.Bar instead", + ): AlsoDeprecated() + with pytest.warns( + MyWarning, + match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar", + ): + class UserClass(AlsoDeprecated): pass - w = self._mywarnings(w) - assert len(w) == 2 - assert "AlsoDeprecated" in str(w[0].message) - assert "foo.Bar" in str(w[0].message) - assert "AlsoDeprecated" in str(w[1].message) - assert "foo.Bar" in str(w[1].message) - def test_inspect_stack(self): with ( mock.patch("inspect.stack", side_effect=IndexError), - warnings.catch_warnings(record=True) as w, + pytest.warns(UserWarning, match="Error detecting parent module"), ): - DeprecatedName = create_deprecated_class("DeprecatedName", NewName) - - class SubClass(DeprecatedName): - pass - - assert "Error detecting parent module" in str(w[0].message) + create_deprecated_class("DeprecatedName", NewName) @mock.patch( @@ -274,12 +262,12 @@ class TestWarnWhenSubclassed: ) class TestUpdateClassPath: def test_old_path_gets_fixed(self): - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + ScrapyDeprecationWarning, + match="`scrapy.contrib.debug.Debug` class is deprecated, use `scrapy.extensions.debug.Debug` instead", + ): output = update_classpath("scrapy.contrib.debug.Debug") assert output == "scrapy.extensions.debug.Debug" - assert len(w) == 1 - assert "scrapy.contrib.debug.Debug" in str(w[0].message) - assert "scrapy.extensions.debug.Debug" in str(w[0].message) def test_sorted_replacement(self): with warnings.catch_warnings(): @@ -288,10 +276,10 @@ class TestUpdateClassPath: assert output == "scrapy.pipelines.Pipeline" def test_unmatched_path_stays_the_same(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) output = update_classpath("scrapy.unmatched.Path") assert output == "scrapy.unmatched.Path" - assert len(w) == 0 def test_returns_nonstring(self): for notastring in [None, True, [1, 2, 3], object()]: diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 9bd86f7fb..0eb330461 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -17,4 +17,4 @@ def test_urlparse_cached(): assert req1a == urlp assert req1a is req1b assert req1a is not req2 - assert req1a is not req2 + assert req1b is not req2 diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index f40e424ff..ee552df64 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -42,33 +42,29 @@ class TestFailureToExcInfo: class TestTopLevelFormatter: - def setup_method(self): - self.handler = LogCapture() - self.handler.addFilter(TopLevelFormatter(["test"])) - - def test_top_level_logger(self): + def test_top_level_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test") - with self.handler as log: - logger.warning("test log msg") - log.check(("test", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples - def test_children_logger(self): + def test_children_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test.test1") - with self.handler as log: - logger.warning("test log msg") - log.check(("test", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples - def test_overlapping_name_logger(self): + def test_overlapping_name_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test2") - with self.handler as log: - logger.warning("test log msg") - log.check(("test2", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test2", logging.WARNING, "test log msg") in caplog.record_tuples - def test_different_name_logger(self): + def test_different_name_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("different") - with self.handler as log: - logger.warning("test log msg") - log.check(("different", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("different", logging.WARNING, "test log msg") in caplog.record_tuples class TestLogCounterHandler: @@ -80,15 +76,16 @@ class TestLogCounterHandler: @pytest.fixture def logger(self, crawler: Crawler) -> Generator[logging.Logger]: logger = logging.getLogger("test") - logger.setLevel(logging.NOTSET) + logger.setLevel(logging.DEBUG) logger.propagate = False - handler = LogCounterHandler(crawler) + handler = LogCounterHandler(crawler, level=crawler.settings.get("LOG_LEVEL")) logger.addHandler(handler) - - yield logger - - logger.propagate = True - logger.removeHandler(handler) + try: + yield logger + finally: + logger.propagate = True + logger.setLevel(logging.NOTSET) + logger.removeHandler(handler) def test_init(self, crawler: Crawler, logger: logging.Logger) -> None: assert crawler.stats @@ -106,7 +103,7 @@ class TestLogCounterHandler: def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None: logger.debug("test log msg") assert crawler.stats - assert crawler.stats.get_value("log_count/INFO") is None + assert crawler.stats.get_value("log_count/DEBUG") is None class TestStreamLogger: @@ -144,7 +141,9 @@ class TestStreamLogger: ], ) def test_spider_logger_adapter_process( - base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict + base_extra: Mapping[str, Any], + log_extra: MutableMapping[str, Any], + expected_extra: dict[str, Any], ) -> None: logger = logging.getLogger("test") spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 610fc0ffe..c4c861404 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -5,6 +5,7 @@ from unittest import mock import pytest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, @@ -13,6 +14,7 @@ from scrapy.utils.misc import ( rel_has_nofollow, set_environ, walk_modules, + walk_modules_iter, ) @@ -38,7 +40,7 @@ class TestUtilsMisc: load_object({}) def test_walk_modules(self): - mods = walk_modules("tests.test_utils_misc.test_walk_modules") + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules") expected = [ "tests.test_utils_misc.test_walk_modules", "tests.test_utils_misc.test_walk_modules.mod", @@ -47,27 +49,38 @@ class TestUtilsMisc: ] assert {m.__name__ for m in mods} == set(expected) - mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod") + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod") expected = [ "tests.test_utils_misc.test_walk_modules.mod", "tests.test_utils_misc.test_walk_modules.mod.mod0", ] assert {m.__name__ for m in mods} == set(expected) - mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1") + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1") expected = [ "tests.test_utils_misc.test_walk_modules.mod1", ] assert {m.__name__ for m in mods} == set(expected) with pytest.raises(ImportError): + for _ in walk_modules_iter("nomodule999"): + pass + with ( + pytest.raises(ImportError), + pytest.warns( + ScrapyDeprecationWarning, + match="The scrapy.utils.misc.walk_modules function is deprecated and will be " + "removed in a future version of Scrapy. " + "Use scrapy.utils.misc.walk_modules_iter instead.", + ), + ): walk_modules("nomodule999") def test_walk_modules_egg(self): egg = str(Path(__file__).parent / "test.egg") sys.path.append(egg) try: - mods = walk_modules("testegg") + mods = walk_modules_iter("testegg") expected = [ "testegg.spiders", "testegg.spiders.a", @@ -147,3 +160,8 @@ class TestUtilsMisc: assert rel_has_nofollow("nofollowfoo") is False assert rel_has_nofollow("foonofollow") is False assert rel_has_nofollow("ugc, , nofollow") is True + # rel attribute values are ASCII case-insensitive per the HTML spec + assert rel_has_nofollow("NoFollow") is True + assert rel_has_nofollow("NOFOLLOW") is True + assert rel_has_nofollow("UGC NoFollow") is True + assert rel_has_nofollow("ugc,NoFollow") is True diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 3783416b9..1acc3aac2 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -93,29 +93,27 @@ https://example.org assert is_generator_with_return_value(h1) assert is_generator_with_return_value(i1) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, + match='The "MockSpider.top_level_return_something" method is a generator', + ): warn_on_generator_with_return_value(mock_spider, top_level_return_something) - assert len(w) == 1 - assert ( - 'The "MockSpider.top_level_return_something" method is a generator' - in str(w[0].message) - ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.f1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, f1) - assert len(w) == 1 - assert 'The "MockSpider.f1" method is a generator' in str(w[0].message) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.g1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, g1) - assert len(w) == 1 - assert 'The "MockSpider.g1" method is a generator' in str(w[0].message) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.h1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, h1) - assert len(w) == 1 - assert 'The "MockSpider.h1" method is a generator' in str(w[0].message) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.i1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, i1) - assert len(w) == 1 - assert 'The "MockSpider.i1" method is a generator' in str(w[0].message) def test_generators_return_none(self, mock_spider): def f2(): @@ -160,32 +158,18 @@ https://example.org assert not is_generator_with_return_value(k2) # not recursive assert not is_generator_with_return_value(l2) - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) warn_on_generator_with_return_value(mock_spider, top_level_return_none) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, f2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, g2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, h2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, i2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, j2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, k2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, l2) - assert len(w) == 0 - def test_generators_return_none_with_decorator(self, mock_spider): # noqa: PLR0915 + def test_generators_return_none_with_decorator(self, mock_spider): def decorator(func): def inner_func(): func() @@ -241,39 +225,23 @@ https://example.org assert not is_generator_with_return_value(k3) # not recursive assert not is_generator_with_return_value(l3) - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) warn_on_generator_with_return_value(mock_spider, top_level_return_none) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, f3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, g3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, h3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, i3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, j3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, k3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, l3) - assert len(w) == 0 @mock.patch( "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error ) def test_indentation_error(self, mock_spider): - with warnings.catch_warnings(record=True) as w: + with pytest.warns(UserWarning, match="Unable to determine"): warn_on_generator_with_return_value(mock_spider, top_level_return_none) - assert len(w) == 1 - assert "Unable to determine" in str(w[0].message) def test_partial(self): def cb(arg1, arg2): @@ -300,13 +268,11 @@ https://example.org yield 1 return "value" - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) warn_on_generator_with_return_value(spider, gen_with_return) - assert len(w) == 0 spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True - with warnings.catch_warnings(record=True) as w: + with pytest.warns(UserWarning, match="is a generator"): warn_on_generator_with_return_value(spider, gen_with_return) - assert len(w) == 1 - assert "is a generator" in str(w[0].message) diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 20a3d940c..5333a55cb 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -1,5 +1,4 @@ import os -import warnings from pathlib import Path import pytest @@ -41,11 +40,8 @@ class TestGetProjectSettings: envvars = { "SCRAPY_SETTINGS_MODULE": value, } - with warnings.catch_warnings(): - warnings.simplefilter("error") - with set_environ(**envvars): - settings = get_project_settings() - + with set_environ(**envvars): + settings = get_project_settings() assert settings.get("SETTINGS_MODULE") == value def test_invalid_envvar(self): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 8dfac5f02..2e8047e2d 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -12,7 +12,6 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import aiter_errback from scrapy.utils.python import ( MutableAsyncChain, - MutableChain, binary_is_text, get_func_args, memoizemethod_noargs, @@ -30,16 +29,6 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -def test_mutablechain(): - m = MutableChain(range(2), [2, 3], (4, 5)) - m.extend(range(6, 7)) - m.extend([7, 8]) - m.extend([9, 10], (11, 12)) - assert next(m) == 0 - assert m.__next__() == 1 - assert list(m) == list(range(2, 13)) - - class TestMutableAsyncChain: @staticmethod async def g1(): @@ -186,7 +175,7 @@ def test_get_func_args(): assert get_func_args(partial_f2) == ["a", "c"] assert get_func_args(partial_f3) == ["c"] assert get_func_args(cal) == ["a", "b", "c"] - assert get_func_args(object) == [] # pylint: disable=use-implicit-booleaness-not-comparison + assert get_func_args(object) == [] assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] assert get_func_args(" ".join, stripself=True) == ["iterable"] diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index 3255d5940..7d39a478e 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -1,5 +1,4 @@ import asyncio -import warnings import pytest @@ -13,23 +12,12 @@ from tests.utils.decorators import coroutine_test class TestAsyncio: - @pytest.mark.requires_reactor + @pytest.mark.requires_reactor # needs a reactor def test_is_asyncio_reactor_installed(self, reactor_pytest: str) -> None: # the result should depend only on the pytest --reactor argument assert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio") - @pytest.mark.requires_reactor - def test_install_asyncio_reactor(self): - from twisted.internet import reactor as original_reactor - - with warnings.catch_warnings(record=True) as w: - install_reactor(_asyncio_reactor_path) - assert len(w) == 0, [str(warning) for warning in w] - from twisted.internet import reactor # pylint: disable=reimported - - assert original_reactor == reactor - - @pytest.mark.requires_reactor + @pytest.mark.requires_reactor # installs a reactor @pytest.mark.only_asyncio @coroutine_test async def test_set_asyncio_event_loop(self): diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index b25129c87..e4967d4e7 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -2,6 +2,7 @@ from __future__ import annotations import json from hashlib import sha1 +from typing import Any from weakref import WeakKeyDictionary import pytest @@ -56,13 +57,17 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None: class TestFingerprint: - function: staticmethod = staticmethod(fingerprint) + function: staticmethod[[Request], bytes] = staticmethod(fingerprint) cache: ( - WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]] - | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]] + WeakKeyDictionary[ + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] + ] + | WeakKeyDictionary[ + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str] + ] ) = _fingerprint_cache - default_cache_key = (None, False) - known_hashes: tuple[tuple[Request, bytes | str, dict], ...] = ( + default_cache_key = (None, False, False) + known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = ( ( Request("http://example.org"), b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", @@ -191,6 +196,32 @@ class TestFingerprint: assert self.function(r2) != self.function(r2, keep_fragments=True) assert self.function(r1) != self.function(r2, keep_fragments=True) + def test_verbatim_url(self): + # verbatim_url requests skip URL canonicalization + r1 = Request( + "http://www.example.com/query?a=1&b=2", meta={"verbatim_url": True} + ) + r2 = Request( + "http://www.example.com/query?b=2&a=1", meta={"verbatim_url": True} + ) + assert self.function(r1) != self.function(r2) + + # without verbatim_url, canonicalization makes query-param order irrelevant + r3 = Request("http://www.example.com/query?a=1&b=2") + r4 = Request("http://www.example.com/query?b=2&a=1") + assert self.function(r3) == self.function(r4) + + # with verbatim_url, the fragment is always kept in the fingerprint + r5 = Request( + "http://www.example.com/test#fragment", meta={"verbatim_url": True} + ) + r6 = Request("http://www.example.com/test", meta={"verbatim_url": True}) + assert self.function(r5) != self.function(r6) + + # keep_fragments parameter is ignored for verbatim_url requests + assert self.function(r5) == self.function(r5, keep_fragments=True) + assert self.function(r5) == self.function(r5, keep_fragments=False) + def test_method_and_body(self): r1 = Request("http://www.example.com") r2 = Request("http://www.example.com", method="POST") @@ -382,3 +413,45 @@ class TestRequestToCurl: " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" ) self._test_request(request_object, expected_curl_command) + + def test_cookies_list_verbose(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies=[ + { + "name": b"foo", + "value": b"bar", + "domain": "example.com", + "path": "/", + "secure": True, + } + ], + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" + ) + self._test_request(request_object, expected_curl_command) + + def test_cookies_list_verbose_non_string_value(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies=[ + { + "name": "foo", + "value": 1, + "domain": "example.com", + "path": "/", + "secure": True, + } + ], + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=1'" + ) + self._test_request(request_object, expected_curl_command) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 0aeb5594f..4544cd29e 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse import pytest -from scrapy.http import HtmlResponse, Response +from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import ( _remove_html_comments, @@ -15,17 +15,21 @@ from scrapy.utils.response import ( ) +def _read_browser_output(burl: str): + path = urlparse(burl).path + if not path or not Path(path).exists(): + path = burl.replace("file://", "") + return Path(path).read_bytes() + + def test_open_in_browser(): - url = "http:///www.example.com/some/page.html" + url = "http://www.example.com/some/page.html" body = ( b" test page test body " ) def browser_open(burl: str) -> bool: - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() + bbody = _read_browser_output(burl) assert b'' in bbody return True @@ -111,36 +115,27 @@ def test_response_status_message(): assert response_status_message(573) == "573 Unknown Status" -def test_inject_base_url(): - url = "http://www.example.com" - - def check_base_url(burl): - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() - assert bbody.count(b'') == 1 - return True - - r1 = HtmlResponse( - url, - body=b""" +@pytest.mark.parametrize( + "body", + [ + pytest.param( + b""" Dummy

Hello world.

""", - ) - r2 = HtmlResponse( - url, - body=b""" + id="Simple", + ), + pytest.param( + b""" Dummy Hello world. """, - ) - r3 = HtmlResponse( - url, - body=b""" + id=" with attrs", + ), + pytest.param( + b""" Dummy @@ -148,19 +143,19 @@ def test_inject_base_url():

Hello world.

""", - ) - r4 = HtmlResponse( - url, - body=b""" + id="Misleading tag", + ), + pytest.param( + b""" Dummy

Hello world.

""", - ) - r5 = HtmlResponse( - url, - body=b""" + id="Misleading comment", + ), + pytest.param( + b"""

Hello world.

""", - ) + id="Conditional comment", + ), + ], +) +def test_inject_base_url(body: bytes) -> None: + url = "http://www.example.com" - assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" - assert open_in_browser(r2, _openfunc=check_base_url), ( - "Inject base url with argumented head" - ) - assert open_in_browser(r3, _openfunc=check_base_url), ( - "Inject unique base url with misleading tag" - ) - assert open_in_browser(r4, _openfunc=check_base_url), ( - "Inject unique base url with misleading comment" - ) - assert open_in_browser(r5, _openfunc=check_base_url), ( - "Inject unique base url with conditional comment" - ) + def check_base_url(burl): + bbody = _read_browser_output(burl) + assert bbody.count(b'>') == 1 + assert b"ccd", b"acd"), (b"ad", b"ad"), + (b"a -->b", b"a -->b"), + (b"real", b"real"), ], ) def test_remove_html_comments(input_body, output_body): assert _remove_html_comments(input_body) == output_body + + +def test_open_in_browser_preserves_html_comments(): + url = "http://www.example.com" + body = ( + b"" + b"" + b"Real" + b"content" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert b"" in bbody + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_does_not_inject_base_when_present(): + url = "http://www.example.com" + body = ( + b"" + b'T' + b"hi" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert b'' not in bbody + assert b'' in bbody + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_injects_base_when_only_in_comment(): + url = "http://www.example.com" + body = ( + b"" + b"" + b"Real" + b"content" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert b'' in bbody + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_injects_base_at_real_head_not_commented_head(): + url = "http://www.example.com" + body = ( + b"" + b"" + b"Actual" + b"hello" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert bbody.count(b'') == 1 + base_pos = bbody.find(b'') + title_pos = bbody.find(b"Actual") + assert base_pos < title_pos + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_text_response_uses_txt_extension(): + response = TextResponse("http://www.example.com", body=b"plain text content") + + def check(burl): + assert burl.endswith(".txt") + return True + + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_raises_for_unsupported_response_type(): + response = Response("http://www.example.com", body=b"binary") + with pytest.raises(TypeError): + open_in_browser(response, _openfunc=lambda _: True) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 464a31777..3599c4824 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -1,3 +1,6 @@ +import pytest + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots @@ -156,7 +159,7 @@ def test_sitemap_wrong_ns2(): def test_sitemap_urls_from_robots(): - robots = """User-agent: * + robots = b"""User-agent: * Disallow: /aff/ Disallow: /wl/ @@ -182,6 +185,40 @@ Disallow: /forum/active/ ] +def test_sitemap_urls_from_robots_str_compat(): + robots = """User-agent: * +Disallow: /aff/ +Disallow: /wl/ + +# Search and shopping refining +Disallow: /s*/*facet +Disallow: /s*/*tags + +# Sitemap files +Sitemap: http://example.com/sitemap.xml +Sitemap: http://example.com/sitemap-product-index.xml +Sitemap: HTTP://example.com/sitemap-uppercase.xml +Sitemap: /sitemap-relative-url.xml + +# Forums +Disallow: /forum/search/ +Disallow: /forum/active/ +""" + + with pytest.warns( + ScrapyDeprecationWarning, + match="Passing `str` type as `robots_text` is deprecated", + ): + assert list( + sitemap_urls_from_robots(robots, base_url="http://example.com") + ) == [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://example.com/sitemap-relative-url.xml", + ] + + def test_sitemap_blanklines(): """Assert we can deal with starting blank lines before tag""" s = Sitemap( diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 3967c3365..7a7a264d3 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,10 +1,11 @@ +import sys from io import StringIO -from time import sleep, time from unittest import mock import pytest from scrapy.utils import trackref +from scrapy.utils.python import garbage_collect class Foo(trackref.object_ref): @@ -63,24 +64,57 @@ Foo 1 oldest: 0s ago\n\n""" ) +_IS_PYPY = "PyPy" in sys.version + + def test_get_oldest(): - o1 = Foo() + """ + Verify that `get_oldest` returns the oldest live instance of a class. - o1_time = time() + The test runs in two passes to expose differences between: + - CPython (reference counting, immediate destruction) + - PyPy (tracing GC, delayed destruction) - o2 = Bar() + Since `trackref` relies on weak references, delayed GC on PyPy can leave + stale entries in `live_refs`, affecting results unless explicitly cleared. + """ - o3_time = time() - if o3_time <= o1_time: - sleep(0.01) - o3_time = time() - if o3_time <= o1_time: - pytest.skip("time.time is not precise enough") + def _delete_o1(): + """Delete `o1` and ensure it is actually collected on PyPy.""" + nonlocal o1 + del o1 - o3 = Foo() # noqa: F841 - assert trackref.get_oldest("Foo") is o1 - assert trackref.get_oldest("Bar") is o2 - assert trackref.get_oldest("XXX") is None + if _IS_PYPY: + # On PyPy, `del` only removes the local reference. The object may + # still exist until the GC runs, so we force a collection cycle. + garbage_collect() + + def _do_asserts(): + assert trackref.get_oldest("Foo") is o1 + assert trackref.get_oldest("Bar") is o2 + # Ensure the newer Foo is not incorrectly considered the oldest + assert trackref.get_oldest("Foo") is not o3 + assert trackref.get_oldest("XXX") is None + + o1, o2, o3 = Foo(), Bar(), Foo() + + _do_asserts() + + # Remove the oldest Foo instance; o3 should now become the oldest + _delete_o1() + assert trackref.get_oldest("Foo") is o3 + + # PyPy-specific behavior where stale references may persist + # unless the registry is explicitly cleared. + if _IS_PYPY: + trackref.live_refs.clear() + + o1, o2, o3 = Foo(), Bar(), Foo() + + _do_asserts() + + _delete_o1() + assert trackref.get_oldest("Foo") is o3 def test_iter_all(): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index bcbbe74a3..a74b9a41d 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,13 +1,9 @@ -import warnings -from importlib import import_module - import pytest from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider -from scrapy.utils.url import ( # type: ignore[attr-defined] +from scrapy.utils.url import ( _is_filesystem_path, - _public_w3lib_objects, add_http_if_no_scheme, guess_scheme, strip_url, @@ -350,6 +346,18 @@ class TestStripUrl: "ftp://username:password@www.example.com:221/file.txt", "ftp://username:password@www.example.com:221/file.txt", ), + ( + "http://user:80@www.example.com:80/index.html", + "http://user:80@www.example.com/index.html", + ), + ( + "https://user:443@www.example.com:443/index.html", + "https://user:443@www.example.com/index.html", + ), + ( + "ftp://user:21@www.example.com:21/file.txt", + "ftp://user:21@www.example.com/file.txt", + ), ], ) def test_default_ports(self, url: str, expected: str) -> None: @@ -446,23 +454,3 @@ class TestStripUrl: ) def test__is_filesystem_path(path: str, expected: bool) -> None: assert _is_filesystem_path(path) == expected - - -@pytest.mark.parametrize( - "obj_name", - [ - "_unquotepath", - "_safe_chars", - "parse_url", - *_public_w3lib_objects, - ], -) -def test_deprecated_imports_from_w3lib(obj_name: str) -> None: - with warnings.catch_warnings(record=True) as warns: - obj_type = "attribute" if obj_name == "_safe_chars" else "function" - message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead." - - getattr(import_module("scrapy.utils.url"), obj_name) - - assert isinstance(warns[0].message, Warning) - assert message in warns[0].message.args diff --git a/tests/test_webclient.py b/tests/test_webclient.py deleted file mode 100644 index 7834e3b31..000000000 --- a/tests/test_webclient.py +++ /dev/null @@ -1,399 +0,0 @@ -""" -Tests borrowed from the twisted.web.client tests. -""" - -from __future__ import annotations - -from urllib.parse import urlparse - -import OpenSSL.SSL -import pytest -from pytest_twisted import async_yield_fixture -from twisted.internet.defer import inlineCallbacks -from twisted.internet.testing import StringTransport -from twisted.protocols.policies import WrappingFactory -from twisted.web import resource, server, static, util -from twisted.web.client import _makeGetterFactory - -from scrapy.core.downloader import webclient as client -from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory -from scrapy.exceptions import DownloadTimeoutError -from scrapy.http import Headers, Request -from scrapy.utils.misc import build_from_crawler -from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.test import get_crawler -from tests.mockserver.http_resources import ( - ForeverTakingResource, - HostHeaderResource, - PayloadResource, -) -from tests.mockserver.utils import ssl_context_factory -from tests.test_core_downloader import TestContextFactoryBase - -pytestmark = pytest.mark.requires_reactor - - -def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): - """Adapted version of twisted.web.client.getPage""" - - def _clientfactory(url, *args, **kwargs): - url = to_unicode(url) - timeout = kwargs.pop("timeout", 0) - f = client.ScrapyHTTPClientFactory( - Request(url, *args, **kwargs), timeout=timeout - ) - f.deferred.addCallback(response_transform or (lambda r: r.body)) - return f - - return _makeGetterFactory( - to_bytes(url), - _clientfactory, - *args, - contextFactory=contextFactory, - **kwargs, - ).deferred - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestScrapyHTTPPageGetter: - def test_earlyHeaders(self): - # basic test stolen from twisted HTTPageGetter - factory = client.ScrapyHTTPClientFactory( - Request( - url="http://foo/bar", - body="some data", - headers={ - "Host": "example.net", - "User-Agent": "fooble", - "Cookie": "blah blah", - "Content-Length": "12981", - "Useful": "value", - }, - ) - ) - - self._test( - factory, - b"GET /bar HTTP/1.0\r\n" - b"Content-Length: 9\r\n" - b"Useful: value\r\n" - b"Connection: close\r\n" - b"User-Agent: fooble\r\n" - b"Host: example.net\r\n" - b"Cookie: blah blah\r\n" - b"\r\n" - b"some data", - ) - - # test minimal sent headers - factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar")) - self._test(factory, b"GET /bar HTTP/1.0\r\nHost: foo\r\n\r\n") - - # test a simple POST with body and content-type - factory = client.ScrapyHTTPClientFactory( - Request( - method="POST", - url="http://foo/bar", - body="name=value", - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - ) - - self._test( - factory, - b"POST /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"Connection: close\r\n" - b"Content-Type: application/x-www-form-urlencoded\r\n" - b"Content-Length: 10\r\n" - b"\r\n" - b"name=value", - ) - - # test a POST method with no body provided - factory = client.ScrapyHTTPClientFactory( - Request(method="POST", url="http://foo/bar") - ) - - self._test( - factory, - b"POST /bar HTTP/1.0\r\nHost: foo\r\nContent-Length: 0\r\n\r\n", - ) - - # test with single and multivalued headers - factory = client.ScrapyHTTPClientFactory( - Request( - url="http://foo/bar", - headers={ - "X-Meta-Single": "single", - "X-Meta-Multivalued": ["value1", "value2"], - }, - ) - ) - - self._test( - factory, - b"GET /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"X-Meta-Multivalued: value1\r\n" - b"X-Meta-Multivalued: value2\r\n" - b"X-Meta-Single: single\r\n" - b"\r\n", - ) - - # same test with single and multivalued headers but using Headers class - factory = client.ScrapyHTTPClientFactory( - Request( - url="http://foo/bar", - headers=Headers( - { - "X-Meta-Single": "single", - "X-Meta-Multivalued": ["value1", "value2"], - } - ), - ) - ) - - self._test( - factory, - b"GET /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"X-Meta-Multivalued: value1\r\n" - b"X-Meta-Multivalued: value2\r\n" - b"X-Meta-Single: single\r\n" - b"\r\n", - ) - - def _test(self, factory, testvalue): - transport = StringTransport() - protocol = client.ScrapyHTTPPageGetter() - protocol.factory = factory - protocol.makeConnection(transport) - assert set(transport.value().splitlines()) == set(testvalue.splitlines()) - return testvalue - - def test_non_standard_line_endings(self): - # regression test for: http://dev.scrapy.org/ticket/258 - factory = client.ScrapyHTTPClientFactory(Request(url="http://foo/bar")) - protocol = client.ScrapyHTTPPageGetter() - protocol.factory = factory - protocol.headers = Headers() - protocol.dataReceived(b"HTTP/1.0 200 OK\n") - protocol.dataReceived(b"Hello: World\n") - protocol.dataReceived(b"Foo: Bar\n") - protocol.dataReceived(b"\n") - assert protocol.headers == Headers({"Hello": ["World"], "Foo": ["Bar"]}) - - -class EncodingResource(resource.Resource): - out_encoding = "cp1251" - - def render(self, request): - body = to_unicode(request.content.read()) - request.setHeader(b"content-encoding", self.out_encoding) - return body.encode(self.out_encoding) - - -class BrokenDownloadResource(resource.Resource): - def render(self, request): - # only sends 3 bytes even though it claims to send 5 - request.setHeader(b"content-length", b"5") - request.write(b"abc") - return b"" - - -class ErrorResource(resource.Resource): - def render(self, request): - request.setResponseCode(401) - if request.args.get(b"showlength"): - request.setHeader(b"content-length", b"0") - return b"" - - -class NoLengthResource(resource.Resource): - def render(self, request): - return b"nolength" - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestWebClient: - def _listen(self, site): - from twisted.internet import reactor - - return reactor.listenTCP(0, site, interface="127.0.0.1") - - @pytest.fixture - def wrapper(self, tmp_path): - (tmp_path / "file").write_bytes(b"0123456789") - r = static.File(str(tmp_path)) - r.putChild(b"redirect", util.Redirect(b"/file")) - r.putChild(b"wait", ForeverTakingResource()) - r.putChild(b"error", ErrorResource()) - r.putChild(b"nolength", NoLengthResource()) - r.putChild(b"host", HostHeaderResource()) - r.putChild(b"payload", PayloadResource()) - r.putChild(b"broken", BrokenDownloadResource()) - r.putChild(b"encoding", EncodingResource()) - site = server.Site(r, timeout=None) - return WrappingFactory(site) - - @async_yield_fixture - async def server_port(self, wrapper): - port = self._listen(wrapper) - - yield port.getHost().port - - await port.stopListening() - - @pytest.fixture - def server_url(self, server_port): - return f"http://127.0.0.1:{server_port}/" - - @inlineCallbacks - def testPayload(self, server_url): - s = "0123456789" * 10 - body = yield getPage(server_url + "payload", body=s) - assert body == to_bytes(s) - - @inlineCallbacks - def testHostHeader(self, server_port, server_url): - # if we pass Host header explicitly, it should be used, otherwise - # it should extract from url - body = yield getPage(server_url + "host") - assert body == to_bytes(f"127.0.0.1:{server_port}") - body = yield getPage(server_url + "host", headers={"Host": "www.example.com"}) - assert body == to_bytes("www.example.com") - - @inlineCallbacks - def test_getPage(self, server_url): - """ - L{client.getPage} returns a L{Deferred} which is called back with - the body of the response if the default method B{GET} is used. - """ - body = yield getPage(server_url + "file") - assert body == b"0123456789" - - @inlineCallbacks - def test_getPageHead(self, server_url): - """ - L{client.getPage} returns a L{Deferred} which is called back with - the empty string if the method is C{HEAD} and there is a successful - response code. - """ - - def _getPage(method): - return getPage(server_url + "file", method=method) - - body = yield _getPage("head") - assert body == b"" - body = yield _getPage("HEAD") - assert body == b"" - - @inlineCallbacks - def test_timeoutNotTriggering(self, server_port, server_url): - """ - When a non-zero timeout is passed to L{getPage} and the page is - retrieved before the timeout period elapses, the L{Deferred} is - called back with the contents of the page. - """ - body = yield getPage(server_url + "host", timeout=100) - assert body == to_bytes(f"127.0.0.1:{server_port}") - - @inlineCallbacks - def test_timeoutTriggering(self, wrapper, server_url): - """ - When a non-zero timeout is passed to L{getPage} and that many - seconds elapse before the server responds to the request. the - L{Deferred} is errbacked with a L{DownloadTimeoutError}. - """ - with pytest.raises(DownloadTimeoutError): - yield getPage(server_url + "wait", timeout=0.000001) - # Clean up the server which is hanging around not doing - # anything. - connected = list(wrapper.protocols.keys()) - # There might be nothing here if the server managed to already see - # that the connection was lost. - if connected: - connected[0].transport.loseConnection() - - @inlineCallbacks - def testNotFound(self, server_url): - body = yield getPage(server_url + "notsuchfile") - assert b"404 - No Such Resource" in body - - @inlineCallbacks - def testFactoryInfo(self, server_url): - from twisted.internet import reactor - - url = server_url + "file" - parsed = urlparse(url) - factory = client.ScrapyHTTPClientFactory(Request(url)) - reactor.connectTCP(parsed.hostname, parsed.port, factory) - yield factory.deferred - assert factory.status == b"200" - assert factory.version.startswith(b"HTTP/") - assert factory.message == b"OK" - assert factory.response_headers[b"content-length"] == b"10" - - @inlineCallbacks - def testRedirect(self, server_url): - body = yield getPage(server_url + "redirect") - assert ( - body - == b'\n\n \n \n' - b' \n \n ' - b'click here\n \n\n' - ) - - @inlineCallbacks - def test_encoding(self, server_url): - """Test that non-standart body encoding matches - Content-Encoding header""" - original_body = b"\xd0\x81\xd1\x8e\xd0\xaf" - response = yield getPage( - server_url + "encoding", body=original_body, response_transform=lambda r: r - ) - content_encoding = to_unicode(response.headers[b"Content-Encoding"]) - assert content_encoding == EncodingResource.out_encoding - assert response.body.decode(content_encoding) == to_unicode(original_body) - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestWebClientSSL(TestContextFactoryBase): - @inlineCallbacks - def testPayload(self, server_url): - s = "0123456789" * 10 - body = yield getPage(server_url + "payload", body=s) - assert body == to_bytes(s) - - -class TestWebClientCustomCiphersSSL(TestWebClientSSL): - # we try to use a cipher that is not enabled by default in OpenSSL - custom_ciphers = "CAMELLIA256-SHA" - context_factory = ssl_context_factory(cipher_string=custom_ciphers) - - @inlineCallbacks - def testPayload(self, server_url): - s = "0123456789" * 10 - crawler = get_crawler( - settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers} - ) - client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) - body = yield getPage( - server_url + "payload", body=s, contextFactory=client_context_factory - ) - assert body == to_bytes(s) - - @inlineCallbacks - def testPayloadDisabledCipher(self, server_url): - s = "0123456789" * 10 - crawler = get_crawler( - settings_dict={ - "DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384" - } - ) - client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) - with pytest.raises(OpenSSL.SSL.Error): - yield getPage( - server_url + "payload", body=s, contextFactory=client_context_factory - ) diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.py index e6ff46f5c..b2ba013f1 100644 --- a/tests/test_zz_resources.py +++ b/tests/test_zz_resources.py @@ -9,12 +9,14 @@ import pytest from scrapy.utils.log import LogCounterHandler from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed +from tests.utils.decorators import coroutine_test def test_counter_handler() -> None: """Test that ``LogCounterHandler`` is always properly removed. - It's added in ``Crawler.crawl{,_async}()`` and removed on engine_stopped. + It's added in ``LogCount.spider_opened()`` and removed in + ``LogCount.spider_closed()``. """ c = sum(1 for h in logging.root.handlers if isinstance(h, LogCounterHandler)) assert c == 0 @@ -25,17 +27,26 @@ def test_stderr_log_handler() -> None: It's added in ``configure_logging()``, called by ``{Async,}CrawlerProcess`` (without ``install_root_handler=False``). It can be removed with - ``_uninstall_scrapy_root_handler()`` if installing it was really neeeded. + ``_uninstall_scrapy_root_handler()`` if installing it was really needed. """ c = sum(1 for h in logging.root.handlers if type(h) is logging.StreamHandler) # pylint: disable=unidiomatic-typecheck assert c == 0 -@pytest.mark.requires_reactor # needs a running event loop for asyncio.all_tasks() @pytest.mark.only_asyncio -def test_pending_asyncio_tasks() -> None: +@coroutine_test +async def test_pending_asyncio_tasks() -> None: """Test that there are no pending asyncio tasks.""" - assert not asyncio.all_tasks() + # note that pytest-asyncio uses separate loops per function so this isn't as useful there + tasks = [] + for t in asyncio.all_tasks(): + coro = t.get_coro() + if ( + coro is not None + and getattr(coro, "__name__", None) != "test_pending_asyncio_tasks" + ): + tasks.append(t) + assert not tasks def test_installed_reactor(reactor_pytest: str) -> None: diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index a81f2d1a2..73c0e3776 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,17 +1,29 @@ +import asyncio import os +from collections.abc import Callable from pathlib import Path from twisted.internet.defer import Deferred +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.defer import maybe_deferred_to_future -def twisted_sleep(seconds): + +def twisted_sleep(seconds: float): from twisted.internet import reactor - d = Deferred() + d: Deferred[None] = Deferred() reactor.callLater(seconds, d.callback, None) return d +async def async_sleep(seconds: float) -> None: + if is_asyncio_available(): + await asyncio.sleep(seconds) + else: + await maybe_deferred_to_future(twisted_sleep(seconds)) + + def get_script_run_env() -> dict[str, str]: """Return a OS environment dict suitable to run scripts shipped with tests.""" @@ -20,3 +32,19 @@ def get_script_run_env() -> dict[str, str]: env = os.environ.copy() env["PYTHONPATH"] = pythonpath return env + + +class OneShotLoop: + """Test stub for create_looping_call: run once immediately, no background task.""" + + def __init__(self, func: Callable[[], None]): + self.func = func + self.running = False + + def start(self, _interval: float, now: bool = True) -> None: + self.running = True + if now: + self.func() + + def stop(self) -> None: + self.running = False diff --git a/tests/utils/decorators.py b/tests/utils/decorators.py index dd9043ab1..0e8988a0a 100644 --- a/tests/utils/decorators.py +++ b/tests/utils/decorators.py @@ -35,7 +35,8 @@ def inline_callbacks_test( async def wrapper_coro(*args: _P.args, **kwargs: _P.kwargs) -> None: await deferred_to_future(inlineCallbacks(f)(*args, **kwargs)) - return wrapper_coro + # Likely https://github.com/python/mypy/issues/17171 + return wrapper_coro # type: ignore[no-any-return] @wraps(f) @inlineCallbacks @@ -55,6 +56,11 @@ def coroutine_test( * with ``pytest-twisted`` this converts a coroutine into a :class:`twisted.internet.defer.Deferred` * with ``pytest-asyncio`` this is a no-op + + In addition to handling asynchronous test functions this can also be used + to mark "synchronous" test functions (they still need to be made + ``async def``) that call code that needs a reactor or a running event loop, + so that ``pytest-asyncio`` starts a loop for them too. """ if not is_reactor_installed(): diff --git a/tests_typing/test_spiders.mypy-testing b/tests_typing/test_spiders.mypy-testing index 162e31d0c..dfdd2ba4d 100644 --- a/tests_typing/test_spiders.mypy-testing +++ b/tests_typing/test_spiders.mypy-testing @@ -59,7 +59,7 @@ def test_spider_parse_override_no_kwargs() -> None: @pytest.mark.mypy_testing def test_spider_parse_override_specific_kwargs() -> None: spider = SpecificKwargsSpider() - reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, page: int) -> Any @pytest.mark.mypy_testing diff --git a/tox.ini b/tox.ini index 1bebaf2aa..2fa7b455d 100644 --- a/tox.ini +++ b/tox.ini @@ -4,13 +4,40 @@ # and then run "tox" from this directory. [tox] -envlist = pre-commit,pylint,typing,py +requires = + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 +envlist = + pre-commit + pylint + mypy + mypy-tests + twinecheck + docs + docs-tests + docs-links + docs-coverage + min + min-extra-deps + min-default-reactor + min-no-reactor + min-botocore + min-pypy3 + py{310,311,312,313,314} + extra-deps + default-reactor + no-reactor + no-reactor-extra-deps + botocore + mitmproxy + pypy3 + pypy3-extra-deps minversion = 1.7.0 [test-requirements] deps = attrs coverage >= 7.10.6 + httpx pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments @@ -38,38 +65,41 @@ download = true commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules} -[testenv:typing] +[testenv:mypy] basepython = python3.10 deps = - mypy==1.19.1 + mypy==2.1.0 typing-extensions==4.15.0 - Pillow==12.1.1 + Pillow==12.2.0 Protego==0.6.0 - attrs==25.4.0 - boto3-stubs[s3]==1.42.59 + Twisted==26.4.0 + attrs==26.1.0 + boto3-stubs[s3]==1.43.9 botocore-stubs==1.42.41 h2==4.3.0 httpx==0.28.1 itemadapter==0.13.1 ptpython==3.0.32 - ipython - pyOpenSSL==25.3.0 - pytest==9.0.2 - types-Pygments==2.19.0.20251121 - types-defusedxml==0.7.0.20250822 + # newer ones require newer Python + ipython==8.39.0 + pyOpenSSL==26.3.0 + pytest==9.0.3 + socksio==1.0.0 + types-Pygments==2.20.0.20260508 + types-defusedxml==0.7.0.20260504 types-lxml==2026.2.16 - types-pexpect==4.9.0.20260127 + types-pexpect==4.9.0.20260508 uvloop==0.22.1 - w3lib==2.4.0 + w3lib==2.4.1 zstandard==0.25.0 commands = mypy {posargs:scrapy tests} -[testenv:typing-tests] +[testenv:mypy-tests] basepython = python3.10 deps = {[test-requirements]deps} - {[testenv:typing]deps} + {[testenv:mypy]deps} pytest-mypy-testing==0.2.0 commands = pytest {posargs:tests_typing} @@ -86,8 +116,9 @@ basepython = python3 deps = {[testenv:extra-deps]deps} pylint==4.0.2 + pylint-per-file-ignores # https://github.com/pylint-dev/pylint/issues/3767#issuecomment-1319916278 commands = - pylint conftest.py docs extras scrapy tests + pylint {posargs:conftest.py docs extras scrapy tests} [testenv:twinecheck] basepython = python3 @@ -98,7 +129,7 @@ commands = python -m build --sdist twine check dist/* -[pinned] +[min] basepython = python3.10 deps = # pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0 @@ -107,28 +138,29 @@ deps = Twisted==21.7.0 cryptography==37.0.0 cssselect==0.9.1 + httpx==0.27.1 itemadapter==0.1.0 lxml==4.6.4 parsel==1.5.0 pyOpenSSL==22.0.0 queuelib==1.4.2 - service_identity==18.1.0 + service_identity==23.1.0 w3lib==1.17.0 zope.interface==5.1.0 {[test-requirements]deps} setenv = - _SCRAPY_PINNED=true + _SCRAPY_MIN=true commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=pinned.junit.xml -o junit_family=legacy --durations=10 scrapy tests} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests} -[testenv:pinned] -basepython = {[pinned]basepython} +[testenv:min] +basepython = {[min]basepython} deps = - {[pinned]deps} + {[min]deps} PyDispatcher==2.0.5 setenv = - {[pinned]setenv} -commands = {[pinned]commands} + {[min]setenv} +commands = {[min]commands} [testenv:extra-deps] basepython = python3 @@ -141,59 +173,65 @@ deps = brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests google-cloud-storage - httpx + httpx[http2,socks] ipython robotexclusionrulesparser uvloop; platform_system != "Windows" and implementation_name != "pypy" zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests -[testenv:extra-deps-pinned] -basepython = {[pinned]basepython} +[testenv:min-extra-deps] +basepython = {[min]basepython} deps = - {[pinned]deps} + {[min]deps} Pillow==8.3.2 + Twisted[http2]==21.7.0 boto3==1.20.0 bpython==0.7.1 brotli==1.2.0; implementation_name != "pypy" brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 - httpx==0.26.0 + httpx==0.27.1 ipython==7.1.0 robotexclusionrulesparser==1.6.2 uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.16.0; implementation_name != "pypy" setenv = - {[pinned]setenv} -commands = {[pinned]commands} + {[min]setenv} +commands = {[min]commands} [testenv:default-reactor] commands = {[testenv]commands} --reactor=default +[testenv:min-default-reactor] +basepython = {[min]basepython} +deps = {[testenv:min]deps} +commands = {[min]commands} --reactor=default +setenv = + {[min]setenv} + [testenv:no-reactor] deps = {[testenv]deps} - httpx pytest-asyncio commands = {[testenv]commands} -p no:twisted --reactor=none -[testenv:default-reactor-pinned] -basepython = {[pinned]basepython} -deps = {[testenv:pinned]deps} -commands = {[pinned]commands} --reactor=default -setenv = - {[pinned]setenv} - -[testenv:no-reactor-pinned] -basepython = {[pinned]basepython} +[testenv:no-reactor-extra-deps] deps = - {[testenv:pinned]deps} - httpx==0.26.0 + {[testenv:extra-deps]deps} pytest-asyncio -commands = {[pinned]commands} -p no:twisted --reactor=none +commands = + {[testenv]commands} -p no:twisted --reactor=none + +[testenv:min-no-reactor] +basepython = {[min]basepython} +deps = + {[testenv:min]deps} + pytest-asyncio +commands = {[min]commands} -p no:twisted --reactor=none setenv = - {[pinned]setenv} + {[min]setenv} [testenv:pypy3] basepython = pypy3 @@ -207,7 +245,7 @@ deps = {[testenv:extra-deps]deps} commands = {[testenv:pypy3]commands} -[testenv:pypy3-pinned] +[testenv:min-pypy3] basepython = pypy3.11 deps = PyPyDispatcher==2.1.0 @@ -222,47 +260,40 @@ deps = parsel==1.5.0 pyOpenSSL==24.3.0 queuelib==1.4.2 - service_identity==18.1.0 - w3lib==1.20.0 + service_identity==23.1.0 + # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses + # an inline flag placement that Python 3.11 treats as an error: global + # flags not at the start of the expression. w3lib 1.18 stopped encoding [] + # in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url + # rely on that encoding. + w3lib==2.1.0 zope.interface==5.1.0 commands = ; disabling coverage pytest {posargs:--durations=10 scrapy tests} setenv = - {[pinned]setenv} + {[min]setenv} -[docs] +[testenv:docs-tests] changedir = docs -deps = - -rdocs/requirements.txt -setenv = - READTHEDOCS_PROJECT=scrapy - READTHEDOCS_VERSION=master - -[testenv:docs] -basepython = python3 -changedir = {[docs]changedir} deps = {[test-requirements]deps} - {[docs]deps} -setenv = {[docs]setenv} + -rdocs/requirements.txt + form2request commands = - sphinx-build -W -b html . {envtmpdir}/html pytest [testenv:docs-coverage] -basepython = python3 -changedir = {[docs]changedir} -deps = {[docs]deps} -setenv = {[docs]setenv} +changedir = docs +deps = + -rdocs/requirements.txt commands = sphinx-build -b coverage . {envtmpdir}/coverage [testenv:docs-links] -basepython = python3 -changedir = {[docs]changedir} -deps = {[docs]deps} -setenv = {[docs]setenv} +changedir = docs +deps = + -rdocs/requirements.txt commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck @@ -276,15 +307,17 @@ deps = commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore -[testenv:botocore-pinned] -basepython = {[pinned]basepython} +[testenv:min-botocore] +basepython = {[min]basepython} deps = - {[pinned]deps} + {[min]deps} botocore==1.13.45 + # botocore 1.13.45 requires urllib3>=1.20,<1.26; requests 2.33.0 requires urllib3>=1.26,<3 + requests<2.33.0 setenv = - {[pinned]setenv} + {[min]setenv} commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy} -m requires_botocore + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore # Run proxy tests that use mitmproxy in a separate env to avoid installing @@ -296,5 +329,6 @@ deps = {[testenv]deps} # mitmproxy does not support PyPy mitmproxy; implementation_name != "pypy" + httpx[http2,socks] commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_mitmproxy + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=mitmproxy.junit.xml -o junit_family=legacy} -m requires_mitmproxy