diff --git a/.bandit.yml b/.bandit.yml index 41f1bb597..2aae8a0aa 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,5 +1,6 @@ skips: - B101 +- B113 # https://github.com/PyCQA/bandit/issues/1010 - B105 - B301 - B303 @@ -17,3 +18,4 @@ skips: - B503 - B603 - B605 +exclude_dirs: ['tests'] diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1d9b9c02f..a00b7cfb3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.6.1 +current_version = 2.9.0 commit = True tag = True tag_name = {new_version} diff --git a/.coveragerc b/.coveragerc index 02acbff8e..ad0ee0f6c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,3 +3,4 @@ branch = true include = scrapy/* omit = tests/* +disable_warnings = include-ignored diff --git a/.flake8 b/.flake8 index 1c503fb0b..544d72956 100644 --- a/.flake8 +++ b/.flake8 @@ -1,19 +1,22 @@ [flake8] max-line-length = 119 -ignore = W503 +ignore = W503, E203 exclude = + docs/conf.py + +per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file # F401: Module imported but unused - scrapy/__init__.py E402 - scrapy/core/downloader/handlers/http.py F401 - scrapy/http/__init__.py F401 - scrapy/linkextractors/__init__.py E402 F401 - scrapy/selector/__init__.py F401 - scrapy/spiders/__init__.py E402 F401 + scrapy/__init__.py:E402 + scrapy/core/downloader/handlers/http.py:F401 + scrapy/http/__init__.py:F401 + scrapy/linkextractors/__init__.py:E402,F401 + scrapy/selector/__init__.py:F401 + scrapy/spiders/__init__.py:E402,F401 # Issues pending a review: - scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 + scrapy/utils/url.py:F403,F405 + tests/test_loader.py:E741 diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000..dbcebfa0a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,7 @@ +# .git-blame-ignore-revs +# adding black formatter to all the code +e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d +# re applying black to the code with default line length +303f0a70fcf8067adf0a909c2096a5009162383a +# reaplying black again and removing line length on pre-commit black config +c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 \ No newline at end of file diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 98fa44c7f..ee0cb4b1e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -3,34 +3,29 @@ on: [push, pull_request] jobs: checks: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - - python-version: "3.10" - env: - TOXENV: security - - python-version: "3.10" - env: - TOXENV: flake8 - # Pylint requires installing reppy, which does not support Python 3.9 - # https://github.com/seomoz/reppy/issues/122 - - python-version: 3.8 + - python-version: "3.11" env: TOXENV: pylint - - python-version: 3.6 + - python-version: 3.8 env: TOXENV: typing - - python-version: "3.10" # Keep in sync with .readthedocs.yml + - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs + - python-version: "3.11" + env: + TOXENV: twinecheck steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -39,3 +34,9 @@ jobs: run: | pip install -U tox tox + + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 44b682830..22b8996b6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,31 +1,21 @@ name: Publish -on: [push] +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' jobs: publish: - runs-on: ubuntu-18.04 - if: startsWith(github.event.ref, 'refs/tags/') - + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - - name: Check Tag - id: check-release-tag - run: | - if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then - echo ::set-output name=release_tag::true - fi - - - name: Publish to PyPI - if: steps.check-release-tag.outputs.release_tag == 'true' - run: | - pip install --upgrade setuptools wheel twine - python setup.py sdist bdist_wheel - export TWINE_USERNAME=__token__ - export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} - twine upload dist/* + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + - run: | + pip install --upgrade build twine + python -m build + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.6.4 + with: + password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 3aaf688c7..3044a1af3 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -3,17 +3,17 @@ on: [push, pull_request] jobs: tests: - runs-on: macos-10.15 + runs-on: macos-11 strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 1fc8d914b..39e3b0af7 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -3,75 +3,68 @@ on: [push, pull_request] jobs: tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: include: - - python-version: 3.7 - env: - TOXENV: py - - python-version: 3.8 - env: - TOXENV: py - python-version: 3.9 env: TOXENV: py - python-version: "3.10" env: TOXENV: py - - python-version: "3.10" + - python-version: "3.11" + env: + TOXENV: py + - python-version: "3.11" env: TOXENV: asyncio - - python-version: pypy3 + - python-version: pypy3.9 env: TOXENV: pypy3 - PYPY_VERSION: 3.6-v7.3.3 # pinned deps - - python-version: 3.6.12 + - python-version: 3.8.17 env: TOXENV: pinned - - python-version: 3.6.12 + - python-version: 3.8.17 env: TOXENV: asyncio-pinned - - python-version: pypy3 + - python-version: pypy3.8 env: TOXENV: pypy3-pinned - PYPY_VERSION: 3.6-v7.2.0 + - python-version: 3.8.17 + env: + TOXENV: extra-deps-pinned + - python-version: 3.8.17 + env: + TOXENV: botocore-pinned - # extras - # extra-deps includes reppy, which does not support Python 3.9 - # https://github.com/seomoz/reppy/issues/122 - - python-version: 3.8 + - python-version: "3.11" env: TOXENV: extra-deps + - python-version: "3.11" + env: + TOXENV: botocore steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4' + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update - # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version - sudo apt-get install libxml2-dev/bionic-updates libxslt-dev + sudo apt-get install libxml2-dev libxslt-dev - name: Run tests env: ${{ matrix.env }} run: | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi pip install -U tox tox diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index ab7385118..5bcf74d5e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -8,15 +8,9 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.6 - env: - TOXENV: windows-pinned - - python-version: 3.7 - env: - TOXENV: py - python-version: 3.8 env: - TOXENV: py + TOXENV: windows-pinned - python-version: 3.9 env: TOXENV: py @@ -26,12 +20,19 @@ jobs: - python-version: "3.10" env: TOXENV: asyncio +# no binary package for lxml for 3.11 yet +# - python-version: "3.11" +# env: +# TOXENV: py +# - python-version: "3.11" +# env: +# TOXENV: asyncio steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index d77d24624..6c5c50e08 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ test-output.* # Windows Thumbs.db + +# OSX miscellaneous +.DS_Store \ No newline at end of file diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000..f238bf7ea --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +profile = black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..31e9ed1ad --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +repos: +- repo: https://github.com/PyCQA/bandit + rev: 1.7.5 + hooks: + - id: bandit + args: [-r, -c, .bandit.yml] +- repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 + hooks: + - id: flake8 +- repo: https://github.com/psf/black.git + rev: 23.3.0 + hooks: + - id: black +- repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort +- repo: https://github.com/adamchainz/blacken-docs + rev: 1.13.0 + hooks: + - id: blacken-docs + additional_dependencies: + - black==23.3.0 diff --git a/.readthedocs.yml b/.readthedocs.yml index 390be3749..e71d34f3a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.10" # Keep in sync with .github/workflows/checks.yml + python: "3.11" # Keep in sync with .github/workflows/checks.yml python: install: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 902cd523e..3c8e4d1b5 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,77 +1,133 @@ + # Contributor Covenant Code of Conduct ## Our Pledge -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to make participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of experience, -nationality, personal appearance, race, religion, or sexual identity and -orientation. +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. ## Our Standards -Examples of behavior that contributes to creating a positive environment -include: +Examples of behavior that contributes to a positive environment for our +community include: -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community -Examples of unacceptable behavior by participants include: +Examples of unacceptable behavior include: -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission +* Publishing others' private information, such as a physical or email address, + without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting -## Our Responsibilities +## Enforcement Responsibilities -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. ## Scope -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@zyte.com. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. +reported to the community leaders responsible for enforcement at +opensource@zyte.com. +All complaints will be reviewed and investigated promptly and fairly. -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at [http://contributor-covenant.org/version/1/4][version]. +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 06e812936..000000000 --- a/INSTALL +++ /dev/null @@ -1,4 +0,0 @@ -For information about installing Scrapy see: - -* docs/intro/install.rst (local file) -* https://docs.scrapy.org/en/latest/intro/install.html (online version) diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 000000000..495413f97 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,4 @@ +For information about installing Scrapy see: + +* [Local docs](docs/intro/install.rst) +* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html) diff --git a/README.rst b/README.rst index 6b563d638..1918850d6 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,5 @@ .. image:: https://scrapy.org/img/scrapylogo.png + :target: https://scrapy.org/ ====== Scrapy @@ -57,13 +58,15 @@ including a list of features. Requirements ============ -* Python 3.6+ +* Python 3.8+ * Works on Linux, Windows, macOS, BSD Install ======= -The quick way:: +The quick way: + +.. code:: bash pip install scrapy @@ -94,8 +97,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details. Code of Conduct --------------- -Please note that this project is released with a Contributor Code of Conduct -(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). +Please note that this project is released with a Contributor `Code of Conduct `_. By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@zyte.com. diff --git a/artwork/README.rst b/artwork/README.rst index 8a1028cde..c1880ef6c 100644 --- a/artwork/README.rst +++ b/artwork/README.rst @@ -2,19 +2,19 @@ Scrapy artwork ============== -This folder contains Scrapy artwork resources such as logos and fonts. +This folder contains the Scrapy artwork resources such as logos and fonts. scrapy-logo.jpg --------------- -Main Scrapy logo, in JPEG format. +The main Scrapy logo, in JPEG format. qlassik.zip ----------- -Font used for Scrapy logo. Homepage: https://www.dafont.com/qlassik.font +The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font scrapy-blog.logo.xcf -------------------- -The logo used in Scrapy blog, in Gimp format. +The logo used in the Scrapy blog, in Gimp format. diff --git a/conftest.py b/conftest.py index 117087790..e1d4b1213 100644 --- a/conftest.py +++ b/conftest.py @@ -4,12 +4,11 @@ import pytest from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor - from tests.keys import generate_keys def _py_files(folder): - return (str(p) for p in Path(folder).rglob('*.py')) + return (str(p) for p in Path(folder).rglob("*.py")) collect_ignore = [ @@ -21,16 +20,16 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with open('tests/ignores.txt') as reader: +with Path("tests/ignores.txt").open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() - if file_path and file_path[0] != '#': + if file_path and file_path[0] != "#": collect_ignore.append(file_path) if not H2_ENABLED: collect_ignore.extend( ( - 'scrapy/core/downloader/handlers/http2.py', + "scrapy/core/downloader/handlers/http2.py", *_py_files("scrapy/core/http2"), ) ) @@ -42,16 +41,6 @@ def chdir(tmpdir): tmpdir.chdir() -def pytest_collection_modifyitems(session, config, items): - # Avoid executing tests when executing `--flake8` flag (pytest-flake8) - try: - from pytest_flake8 import Flake8Item - if config.getoption('--flake8'): - items[:] = [item for item in items if isinstance(item, Flake8Item)] - except ImportError: - pass - - def pytest_addoption(parser): parser.addoption( "--reactor", @@ -60,7 +49,7 @@ def pytest_addoption(parser): ) -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def reactor_pytest(request): if not request.cls: # doctests @@ -71,14 +60,17 @@ def reactor_pytest(request): @pytest.fixture(autouse=True) def only_asyncio(request, reactor_pytest): - if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio': - pytest.skip('This test is only run with --reactor=asyncio') + if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": + pytest.skip("This test is only run with --reactor=asyncio") @pytest.fixture(autouse=True) def only_not_asyncio(request, reactor_pytest): - if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio': - pytest.skip('This test is only run without --reactor=asyncio') + if ( + request.node.get_closest_marker("only_not_asyncio") + and reactor_pytest == "asyncio" + ): + pytest.skip("This test is only run without --reactor=asyncio") def pytest_configure(config): diff --git a/docs/Makefile b/docs/Makefile index 87d5d3047..48401bac8 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -86,8 +86,8 @@ coverage: BUILDER = coverage coverage: build htmlview: html - $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \ - os.path.realpath('build/html/index.html'))" + $(PYTHON) -c "import webbrowser; from pathlib import Path; \ + webbrowser.open(Path('build/html/index.html').resolve().as_uri())" clean: -rm -rf build/* diff --git a/docs/README.rst b/docs/README.rst index 0b7afa548..36dd5aea4 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your Start over ---------- -To cleanup all generated documentation files and start from scratch run:: +To clean up all generated documentation files and start from scratch run:: make clean diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 640660943..c23a89089 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,8 +1,9 @@ -from docutils.parsers.rst.roles import set_classes +from operator import itemgetter + from docutils import nodes from docutils.parsers.rst import Directive +from docutils.parsers.rst.roles import set_classes from sphinx.util.nodes import make_refnode -from operator import itemgetter class settingslist_node(nodes.General, nodes.Element): @@ -11,15 +12,15 @@ class settingslist_node(nodes.General, nodes.Element): class SettingsListDirective(Directive): def run(self): - return [settingslist_node('')] + return [settingslist_node("")] def is_setting_index(node): - if node.tagname == 'index': + if node.tagname == "index" and node["entries"]: # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] - entry_type, info, refid = node['entries'][0][:3] - return entry_type == 'pair' and info.endswith('; setting') + entry_type, info, refid = node["entries"][0][:3] + return entry_type == "pair" and info.endswith("; setting") return False @@ -30,14 +31,14 @@ def get_setting_target(node): def get_setting_name_and_refid(node): """Extract setting name from directive index node""" - entry_type, info, refid = node['entries'][0][:3] - return info.replace('; setting', ''), refid + entry_type, info, refid = node["entries"][0][:3] + return info.replace("; setting", ""), refid def collect_scrapy_settings_refs(app, doctree): env = app.builder.env - if not hasattr(env, 'scrapy_all_settings'): + if not hasattr(env, "scrapy_all_settings"): env.scrapy_all_settings = [] for node in doctree.traverse(is_setting_index): @@ -46,18 +47,23 @@ def collect_scrapy_settings_refs(app, doctree): setting_name, refid = get_setting_name_and_refid(node) - env.scrapy_all_settings.append({ - 'docname': env.docname, - 'setting_name': setting_name, - 'refid': refid, - }) + env.scrapy_all_settings.append( + { + "docname": env.docname, + "setting_name": setting_name, + "refid": refid, + } + ) def make_setting_element(setting_data, app, fromdocname): - refnode = make_refnode(app.builder, fromdocname, - todocname=setting_data['docname'], - targetid=setting_data['refid'], - child=nodes.Text(setting_data['setting_name'])) + refnode = make_refnode( + app.builder, + fromdocname, + todocname=setting_data["docname"], + targetid=setting_data["refid"], + child=nodes.Text(setting_data["setting_name"]), + ) p = nodes.paragraph() p += refnode @@ -71,69 +77,72 @@ def replace_settingslist_nodes(app, doctree, fromdocname): for node in doctree.traverse(settingslist_node): settings_list = nodes.bullet_list() - settings_list.extend([make_setting_element(d, app, fromdocname) - for d in sorted(env.scrapy_all_settings, - key=itemgetter('setting_name')) - if fromdocname != d['docname']]) + settings_list.extend( + [ + make_setting_element(d, app, fromdocname) + for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) + if fromdocname != d["docname"] + ] + ) node.replace_self(settings_list) def setup(app): app.add_crossref_type( - directivename = "setting", - rolename = "setting", - indextemplate = "pair: %s; setting", + directivename="setting", + rolename="setting", + indextemplate="pair: %s; setting", ) app.add_crossref_type( - directivename = "signal", - rolename = "signal", - indextemplate = "pair: %s; signal", + directivename="signal", + rolename="signal", + indextemplate="pair: %s; signal", ) app.add_crossref_type( - directivename = "command", - rolename = "command", - indextemplate = "pair: %s; command", + directivename="command", + rolename="command", + indextemplate="pair: %s; command", ) app.add_crossref_type( - directivename = "reqmeta", - rolename = "reqmeta", - indextemplate = "pair: %s; reqmeta", + directivename="reqmeta", + rolename="reqmeta", + indextemplate="pair: %s; reqmeta", ) - app.add_role('source', source_role) - app.add_role('commit', commit_role) - app.add_role('issue', issue_role) - app.add_role('rev', rev_role) + app.add_role("source", source_role) + app.add_role("commit", commit_role) + app.add_role("issue", issue_role) + app.add_role("rev", rev_role) app.add_node(settingslist_node) - app.add_directive('settingslist', SettingsListDirective) + app.add_directive("settingslist", SettingsListDirective) - app.connect('doctree-read', collect_scrapy_settings_refs) - app.connect('doctree-resolved', replace_settingslist_nodes) + app.connect("doctree-read", collect_scrapy_settings_refs) + app.connect("doctree-resolved", replace_settingslist_nodes) def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = 'https://github.com/scrapy/scrapy/blob/master/' + text + ref = "https://github.com/scrapy/scrapy/blob/master/" + text set_classes(options) node = nodes.reference(rawtext, text, refuri=ref, **options) return [node], [] def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = 'https://github.com/scrapy/scrapy/issues/' + text + ref = "https://github.com/scrapy/scrapy/issues/" + text set_classes(options) - node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options) + node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options) return [node], [] def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = 'https://github.com/scrapy/scrapy/commit/' + text + ref = "https://github.com/scrapy/scrapy/commit/" + text set_classes(options) - node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options) + node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options) return [node], [] def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = 'http://hg.scrapy.org/scrapy/changeset/' + text + ref = "http://hg.scrapy.org/scrapy/changeset/" + text set_classes(options) - node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options) + node = nodes.reference(rawtext, "r" + text, refuri=ref, **options) return [node], [] diff --git a/docs/_static/selectors-sample1.html b/docs/_static/selectors-sample1.html index 8a79a3381..915718832 100644 --- a/docs/_static/selectors-sample1.html +++ b/docs/_static/selectors-sample1.html @@ -1,16 +1,17 @@ - - - - Example website - - -
- Name: My image 1
- Name: My image 2
- Name: My image 3
- Name: My image 4
- Name: My image 5
-
- - + + + + + Example website + + +
+ Name: My image 1
image1 + Name: My image 2
image2 + Name: My image 3
image3 + Name: My image 4
image4 + Name: My image 5
image5 +
+ + \ No newline at end of file diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html deleted file mode 100644 index 18a5231ee..000000000 --- a/docs/_templates/layout.html +++ /dev/null @@ -1,11 +0,0 @@ -{% extends "!layout.html" %} - -{% block footer %} -{{ super() }} - -{% endblock %} diff --git a/docs/conf.py b/docs/conf.py index 55aa72d5a..38ca81932 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,13 +11,12 @@ import sys from datetime import datetime -from os import path +from pathlib import Path # If your extensions are in another directory, add it here. If the directory -# is relative to the documentation root, use os.path.abspath to make it -# absolute, like shown here. -sys.path.append(path.join(path.dirname(__file__), "_ext")) -sys.path.insert(0, path.dirname(path.dirname(__file__))) +# is relative to the documentation root, use Path.absolute to make it absolute. +sys.path.append(str(Path(__file__).parent / "_ext")) +sys.path.insert(0, str(Path(__file__).parent.parent)) # General configuration @@ -26,30 +25,30 @@ sys.path.insert(0, path.dirname(path.dirname(__file__))) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - 'hoverxref.extension', - 'notfound.extension', - 'scrapydocs', - 'sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.intersphinx', - 'sphinx.ext.viewcode', + "hoverxref.extension", + "notfound.extension", + "scrapydocs", + "sphinx.ext.autodoc", + "sphinx.ext.coverage", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8' +# source_encoding = 'utf-8' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'Scrapy' -copyright = f'2008–{datetime.now().year}, Scrapy developers' +project = "Scrapy" +copyright = f"2008–{datetime.now().year}, Scrapy developers" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -58,50 +57,51 @@ copyright = f'2008–{datetime.now().year}, Scrapy developers' # The short X.Y version. try: import scrapy - version = '.'.join(map(str, scrapy.version_info[:2])) + + version = ".".join(map(str, scrapy.version_info[:2])) release = scrapy.__version__ except ImportError: - version = '' - release = '' + version = "" + release = "" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -language = 'en' +language = "en" # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. -#unused_docs = [] +# unused_docs = [] -exclude_patterns = ['build'] +exclude_patterns = ["build"] # List of directories, relative to source directory, that shouldn't be searched # for source files. -exclude_trees = ['.build'] +exclude_trees = [".build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # List of Sphinx warnings that will not be raised -suppress_warnings = ['epub.unknown_project_files'] +suppress_warnings = ["epub.unknown_project_files"] # Options for HTML output @@ -109,17 +109,18 @@ suppress_warnings = ['epub.unknown_project_files'] # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # Add path to the RTD explicitly to robustify builds (otherwise might # fail in a clean Debian build env) import sphinx_rtd_theme + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The style sheet to use for HTML and HTML Help pages. A file of that name @@ -129,44 +130,44 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = "%b %d, %Y" # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_use_modindex = True +# html_use_modindex = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, the reST sources are included in the HTML build as _sources/. html_copy_source = True @@ -174,16 +175,16 @@ html_copy_source = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = '' +# html_file_suffix = '' # Output file base name for HTML help builder. -htmlhelp_basename = 'Scrapydoc' +htmlhelp_basename = "Scrapydoc" html_css_files = [ - 'custom.css', + "custom.css", ] @@ -191,34 +192,33 @@ html_css_files = [ # ------------------------ # The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +# latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +# latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ('index', 'Scrapy.tex', 'Scrapy Documentation', - 'Scrapy developers', 'manual'), + ("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # Additional stuff for the LaTeX preamble. -#latex_preamble = '' +# latex_preamble = '' # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_use_modindex = True +# latex_use_modindex = True # Options for the linkcheck builder @@ -227,8 +227,9 @@ latex_documents = [ # A list of regular expressions that match URIs that should not be checked when # doing a linkcheck build. linkcheck_ignore = [ - 'http://localhost:\d+', 'http://hg.scrapy.org', - 'http://directory.google.com/' + "http://localhost:\d+", + "http://hg.scrapy.org", + "http://directory.google.com/", ] @@ -238,44 +239,35 @@ coverage_ignore_pyobjects = [ # Contract’s add_pre_hook and add_post_hook are not documented because # they should be transparent to contract developers, for whom pre_hook and # post_hook should be the actual concern. - r'\bContract\.add_(pre|post)_hook$', - + r"\bContract\.add_(pre|post)_hook$", # ContractsManager is an internal class, developers are not expected to # interact with it directly in any way. - r'\bContractsManager\b$', - + r"\bContractsManager\b$", # For default contracts we only want to document their general purpose in # their __init__ method, the methods they reimplement to achieve that purpose # should be irrelevant to developers using those contracts. - r'\w+Contract\.(adjust_request_args|(pre|post)_process)$', - + r"\w+Contract\.(adjust_request_args|(pre|post)_process)$", # Methods of downloader middlewares are not documented, only the classes # themselves, since downloader middlewares are controlled through Scrapy # settings. - r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.', - + r"^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.", # Base classes of downloader middlewares are implementation details that # are not meant for users. - r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware', - + r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware", # Private exception used by the command-line interface implementation. - r'^scrapy\.exceptions\.UsageError', - + r"^scrapy\.exceptions\.UsageError", # Methods of BaseItemExporter subclasses are only documented in # BaseItemExporter. - r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.', - + r"^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.", # Extension behavior is only modified through settings. Methods of # extension classes, as well as helper functions, are implementation # details that are not documented. - r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.', # methods - r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions - + r"^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.", # methods + r"^scrapy\.extensions\.[a-z]\w*?\.[a-z]", # helper functions # Never documented before, and deprecated now. - r'^scrapy\.linkextractors\.FilteringLinkExtractor$', - + r"^scrapy\.linkextractors\.FilteringLinkExtractor$", # Implementation detail of LxmlLinkExtractor - r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor', + r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor", ] @@ -283,18 +275,20 @@ coverage_ignore_pyobjects = [ # ------------------------------------- intersphinx_mapping = { - 'attrs': ('https://www.attrs.org/en/stable/', None), - 'coverage': ('https://coverage.readthedocs.io/en/stable', None), - 'cryptography' : ('https://cryptography.io/en/latest/', None), - 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), - 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), - 'pytest': ('https://docs.pytest.org/en/latest', None), - 'python': ('https://docs.python.org/3', None), - 'sphinx': ('https://www.sphinx-doc.org/en/master', None), - 'tox': ('https://tox.readthedocs.io/en/latest', None), - 'twisted': ('https://twistedmatrix.com/documents/current', None), - 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None), + "attrs": ("https://www.attrs.org/en/stable/", None), + "coverage": ("https://coverage.readthedocs.io/en/stable", None), + "cryptography": ("https://cryptography.io/en/latest/", None), + "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), + "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), + "pytest": ("https://docs.pytest.org/en/latest", None), + "python": ("https://docs.python.org/3", None), + "sphinx": ("https://www.sphinx-doc.org/en/master", None), + "tox": ("https://tox.wiki/en/latest/", None), + "twisted": ("https://docs.twisted.org/en/stable/", None), + "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), + "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), } +intersphinx_disabled_reftypes = [] # Options for sphinx-hoverxref options @@ -312,16 +306,16 @@ hoverxref_role_types = { "setting": "tooltip", "signal": "tooltip", } -hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal'] +hoverxref_roles = ["command", "reqmeta", "setting", "signal"] def setup(app): - app.connect('autodoc-skip-member', maybe_skip_member) + app.connect("autodoc-skip-member", maybe_skip_member) def maybe_skip_member(app, what, name, obj, skip, options): if not skip: # autodocs was generating a text "alias of" for the following members # https://github.com/sphinx-doc/sphinx/issues/4422 - return name in {'default_item_class', 'default_selector_class'} + return name in {"default_item_class", "default_selector_class"} return skip diff --git a/docs/conftest.py b/docs/conftest.py index a0636f8ac..32f849a36 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -1,33 +1,34 @@ -import os from doctest import ELLIPSIS, NORMALIZE_WHITESPACE +from pathlib import Path -from scrapy.http.response.html import HtmlResponse from sybil import Sybil +from sybil.parsers.doctest import DocTestParser +from sybil.parsers.skip import skip + try: # >2.0.1 from sybil.parsers.codeblock import PythonCodeBlockParser except ImportError: from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser -from sybil.parsers.doctest import DocTestParser -from sybil.parsers.skip import skip + +from scrapy.http.response.html import HtmlResponse -def load_response(url, filename): - input_path = os.path.join(os.path.dirname(__file__), '_tests', filename) - with open(input_path, 'rb') as input_file: - return HtmlResponse(url, body=input_file.read()) +def load_response(url: str, filename: str) -> HtmlResponse: + input_path = Path(__file__).parent / "_tests" / filename + return HtmlResponse(url, body=input_path.read_bytes()) def setup(namespace): - namespace['load_response'] = load_response + namespace["load_response"] = load_response pytest_collect_file = Sybil( parsers=[ DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE), - PythonCodeBlockParser(future_imports=['print_function']), + PythonCodeBlockParser(future_imports=["print_function"]), skip, ], - pattern='*.rst', + pattern="*.rst", setup=setup, ).pytest() diff --git a/docs/contributing.rst b/docs/contributing.rst index 4d2580a6c..2b3249601 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -11,10 +11,6 @@ Contributing to Scrapy There are many ways to contribute to Scrapy. Here are some of them: -* Blog about Scrapy. Tell the world how you're using Scrapy. This will help - newcomers with more examples and will help the Scrapy project to increase its - visibility. - * Report bugs and request features in the `issue tracker`_, trying to follow the guidelines detailed in `Reporting bugs`_ below. @@ -22,13 +18,16 @@ There are many ways to contribute to Scrapy. Here are some of them: :ref:`writing-patches` and `Submitting patches`_ below for details on how to write and submit a patch. +* Blog about Scrapy. Tell the world how you're using Scrapy. This will help + newcomers with more examples and will help the Scrapy project to increase its + visibility. + * Join the `Scrapy subreddit`_ and share your ideas on how to improve Scrapy. We're always open to suggestions. * Answer Scrapy questions at `Stack Overflow `__. - Reporting bugs ============== @@ -49,7 +48,7 @@ guidelines when you're going to report a new bug. (use "scrapy" tag). * check the `open issues`_ to see if the issue has already been reported. If it - has, don't dismiss the report, but check the ticket history and comments. If + has, don't dismiss the report, but check the ticket history and comments. If you have additional useful information, please leave a comment, or consider :ref:`sending a pull request ` with a fix. @@ -80,6 +79,13 @@ guidelines when you're going to report a new bug. Writing patches =============== +Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you +can work on. These issues are a great way to get started with contributing to +Scrapy. If you're new to the codebase, you may want to focus on documentation +or testing-related issues, as they are always useful and can help you get +more familiar with the project. You can also check Scrapy's `test coverage`_ +to see which areas may benefit from more tests. + The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged. Well-written patches should: @@ -169,16 +175,43 @@ Coding style Please follow these coding conventions when writing code for inclusion in Scrapy: -* Unless otherwise specified, follow :pep:`8`. - -* It's OK to use lines longer than 79 chars if it improves the code - readability. +* We use `black `_ for code formatting. + There is a hook in the pre-commit config + that will automatically format your code before every commit. You can also + run black manually with ``tox -e black``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. See https://help.github.com/en/github/using-git/setting-your-username-in-git for setup instructions. +.. _scrapy-pre-commit: + +Pre-commit +========== + +We use `pre-commit`_ to automatically address simple code issues before every +commit. + +.. _pre-commit: https://pre-commit.com/ + +After your create a local clone of your fork of the Scrapy repository: + +#. `Install pre-commit `_. + +#. On the root of your local clone of the Scrapy repository, run the following + command: + + .. code-block:: bash + + pre-commit install + +Now pre-commit will check your changes every time you create a Git commit. Upon +finding issues, pre-commit aborts your commit, and either fixes those issues +automatically, or only reports them to you. If it fixes those issues +automatically, creating your commit again should succeed. Otherwise, you may +need to address the corresponding issues manually first. + .. _documentation-policies: Documentation policies @@ -214,7 +247,7 @@ Tests ===== Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires +`. Running tests requires :doc:`tox `. .. _running-tests: @@ -232,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use: To run the tests on a specific :doc:`tox ` environment, use ``-e `` with an environment name from ``tox.ini``. For example, to run -the tests with Python 3.6 use:: +the tests with Python 3.10 use:: - tox -e py36 + tox -e py310 You can also specify a comma-separated list of environments, and use :ref:`tox’s parallel mode ` to run the tests on multiple environments in parallel:: - tox -e py36,py38 -p auto + tox -e py39,py310 -p auto To pass command-line options to :doc:`pytest `, add them after ``--`` in your call to :doc:`tox `. Using ``--`` overrides the @@ -250,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.6 :doc:`tox ` environment using all your CPU cores:: +the Python 3.10 :doc:`tox ` environment using all your CPU cores:: - tox -e py36 -- scrapy tests -n auto + tox -e py310 -- scrapy tests -n auto To see coverage report install :doc:`coverage ` (``pip install coverage``) and run: @@ -287,3 +320,6 @@ And their unit-tests are in:: .. _PEP 257: https://www.python.org/dev/peps/pep-0257/ .. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist +.. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 +.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 +.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy diff --git a/docs/faq.rst b/docs/faq.rst index 8a9ba809b..20dd814df 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -35,8 +35,10 @@ for parsing HTML responses in Scrapy callbacks. You just have to feed the response's body into a ``BeautifulSoup`` object and extract whatever data you need from it. -Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser:: +Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser: +.. skip: next +.. code-block:: python from bs4 import BeautifulSoup import scrapy @@ -45,17 +47,12 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars class ExampleSpider(scrapy.Spider): name = "example" allowed_domains = ["example.com"] - start_urls = ( - 'http://www.example.com/', - ) + start_urls = ("http://www.example.com/",) def parse(self, response): # use lxml to get decent HTML parsing speed - soup = BeautifulSoup(response.text, 'lxml') - yield { - "url": response.url, - "title": soup.h1.string - } + soup = BeautifulSoup(response.text, "lxml") + yield {"url": response.url, "title": soup.h1.string} .. note:: @@ -109,11 +106,13 @@ basically means that it crawls in `DFO order`_. This order is more convenient in most cases. If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings:: +setting the following settings: + +.. code-block:: python DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue' - SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue' + SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" + SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" While pending requests are below the configured values of :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or @@ -159,11 +158,13 @@ See also other suggestions at `StackOverflow`_. .. note:: Remember to disable :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable - your custom implementation:: + your custom implementation: + + .. code-block:: python SPIDER_MIDDLEWARES = { - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, - 'myproject.middlewares.CustomOffsiteMiddleware': 500, + "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, + "myproject.middlewares.CustomOffsiteMiddleware": 500, } .. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation @@ -230,16 +231,20 @@ Can I return (Twisted) deferreds from signal handlers? Some signals support returning deferreds from their handlers, others don't. See the :ref:`topics-signals-ref` to know which ones. -What does the response status code 999 means? ---------------------------------------------- +What does the response status code 999 mean? +-------------------------------------------- 999 is a custom response status code used by Yahoo sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or -higher) in your spider:: +higher) in your spider: + +.. code-block:: python + + from scrapy.spiders import CrawlSpider + class MySpider(CrawlSpider): - - name = 'myspider' + name = "myspider" download_delay = 2 @@ -351,19 +356,21 @@ How to split an item into multiple items in an item pipeline? input item. :ref:`Create a spider middleware ` instead, and use its :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` -method for this purpose. For example:: +method for this purpose. For example: + +.. code-block:: python from copy import deepcopy from itemadapter import is_item, ItemAdapter - class MultiplyItemsMiddleware: + class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): for item in result: if is_item(item): adapter = ItemAdapter(item) - for _ in range(adapter['multiply_by']): + for _ in range(adapter["multiply_by"]): yield deepcopy(item) Does Scrapy support IPv6 addresses? @@ -413,4 +420,4 @@ See :issue:`2680`. .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search \ No newline at end of file +.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/index.rst b/docs/index.rst index 75e08f537..5404969e0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -130,7 +130,6 @@ Built-in services topics/stats topics/email topics/telnetconsole - topics/webservice :doc:`topics/logging` Learn how to use Python's builtin logging on Scrapy. @@ -144,9 +143,6 @@ Built-in services :doc:`topics/telnetconsole` Inspect a running crawler using a built-in Python console. -:doc:`topics/webservice` - Monitor and control a crawler using a web service. - Solving specific problems ========================= @@ -229,10 +225,11 @@ Extending Scrapy topics/downloader-middleware topics/spider-middleware topics/extensions - topics/api topics/signals topics/scheduler topics/exporters + topics/components + topics/api :doc:`topics/architecture` @@ -247,9 +244,6 @@ Extending Scrapy :doc:`topics/extensions` Extend Scrapy with your custom functionality -:doc:`topics/api` - Use it on extensions and middlewares to extend Scrapy functionality - :doc:`topics/signals` See all available signals and how to work with them. @@ -259,6 +253,13 @@ Extending Scrapy :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). +:doc:`topics/components` + Learn the common API and some good practices when building custom Scrapy + components. + +:doc:`topics/api` + Use it on extensions and middlewares to extend Scrapy functionality. + All the rest ============ diff --git a/docs/intro/install.rst b/docs/intro/install.rst index b8d3a16bc..c90c1d2bf 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,8 +9,8 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.6+, either the CPython implementation (default) or -the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). +Scrapy requires Python 3.8+, either the CPython implementation (default) or +the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: @@ -52,17 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among * `twisted`_, an asynchronous networking framework * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs -The minimal versions which Scrapy is tested against are: - -* Twisted 14.0 -* lxml 3.4 -* pyOpenSSL 0.14 - -Scrapy may work with older versions of these packages -but it is not guaranteed it will continue working -because it’s not being tested against them. - -Some of these packages themselves depends on non-Python packages +Some of these packages themselves depend on non-Python packages that might require additional installation steps depending on your platform. Please check :ref:`platform-specific guides below `. @@ -197,7 +187,7 @@ solutions: * Install `homebrew`_ following the instructions in https://brew.sh/ * Update your ``PATH`` variable to state that homebrew packages should be - used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly + used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly if you're using `zsh`_ as default shell):: echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc @@ -229,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy:: PyPy ---- -We recommend using the latest PyPy version. The version tested is 5.9.0. +We recommend using the latest PyPy version. For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index f3d652621..542760b4f 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -20,22 +20,24 @@ In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. Here's the code for a spider that scrapes famous quotes from website -https://quotes.toscrape.com, following the pagination:: +https://quotes.toscrape.com, following the pagination: + +.. code-block:: python import scrapy class QuotesSpider(scrapy.Spider): - name = 'quotes' + name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/tag/humor/', + "https://quotes.toscrape.com/tag/humor/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'author': quote.xpath('span/small/text()').get(), - 'text': quote.css('span.text::text').get(), + "author": quote.xpath("span/small/text()").get(), + "text": quote.css("span.text::text").get(), } next_page = response.css('li.next a::attr("href")').get() @@ -45,9 +47,9 @@ https://quotes.toscrape.com, following the pagination:: Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider quotes_spider.py -o quotes.jl + scrapy runspider quotes_spider.py -o quotes.jsonl -When this finishes you will have in the ``quotes.jl`` file a list of the +When this finishes you will have in the ``quotes.jsonl`` file a list of the quotes in JSON Lines format, containing text and author, looking like this:: {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index cde1b1ef4..19a76fc16 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -83,7 +83,11 @@ optionally how to follow links in the pages, and how to parse the downloaded page content to extract data. This is the code for our first Spider. Save it in a file named -``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:: +``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project: + +.. code-block:: python + + from pathlib import Path import scrapy @@ -93,18 +97,17 @@ This is the code for our first Spider. Save it in a file named def start_requests(self): urls = [ - 'https://quotes.toscrape.com/page/1/', - 'https://quotes.toscrape.com/page/2/', + "https://quotes.toscrape.com/page/1/", + "https://quotes.toscrape.com/page/2/", ] for url in urls: yield scrapy.Request(url=url, callback=self.parse) def parse(self, response): page = response.url.split("/")[-2] - filename = f'quotes-{page}.html' - with open(filename, 'wb') as f: - f.write(response.body) - self.log(f'Saved file {filename}') + filename = f"quotes-{page}.html" + Path(filename).write_bytes(response.body) + self.log(f"Saved file {filename}") As you can see, our Spider subclasses :class:`scrapy.Spider ` @@ -176,7 +179,11 @@ that generates :class:`scrapy.Request ` objects from URLs, you can just define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider:: +for your spider. + +.. code-block:: python + + from pathlib import Path import scrapy @@ -184,15 +191,14 @@ for your spider:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', - 'https://quotes.toscrape.com/page/2/', + "https://quotes.toscrape.com/page/1/", + "https://quotes.toscrape.com/page/2/", ] def parse(self, response): page = response.url.split("/")[-2] - filename = f'quotes-{page}.html' - with open(filename, 'wb') as f: - f.write(response.body) + filename = f"quotes-{page}.html" + Path(filename).write_bytes(response.body) The :meth:`~scrapy.Spider.parse` method will be called to handle each of the requests for those URLs, even though we haven't explicitly told Scrapy @@ -243,8 +249,10 @@ object: response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html') ->>> response.css('title') -[] +.. code-block:: pycon + + >>> response.css("title") + [] The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of @@ -254,42 +262,54 @@ data. To extract the text from the title above, you can do: ->>> response.css('title::text').getall() -['Quotes to Scrape'] +.. code-block:: pycon + + >>> response.css("title::text").getall() + ['Quotes to Scrape'] There are two things to note here: one is that we've added ``::text`` to the CSS query, to mean we want to select only the text elements directly inside ```` element. If we don't specify ``::text``, we'd get the full title element, including its tags: ->>> response.css('title').getall() -['<title>Quotes to Scrape'] +.. code-block:: pycon + + >>> response.css("title").getall() + ['Quotes to Scrape'] The other thing is that the result of calling ``.getall()`` is a list: it is possible that a selector returns more than one result, so we extract them all. When you know you just want the first result, as in this case, you can do: ->>> response.css('title::text').get() -'Quotes to Scrape' +.. code-block:: pycon + + >>> response.css("title::text").get() + 'Quotes to Scrape' As an alternative, you could've written: ->>> response.css('title::text')[0].get() -'Quotes to Scrape' +.. code-block:: pycon + + >>> response.css("title::text")[0].get() + 'Quotes to Scrape' Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will -raise an :exc:`IndexError` exception if there are no results:: +raise an :exc:`IndexError` exception if there are no results: - >>> response.css('noelement')[0].get() +.. code-block:: pycon + + >>> response.css("noelement")[0].get() Traceback (most recent call last): ... IndexError: list index out of range You might want to use ``.get()`` directly on the :class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` -if there are no results:: +if there are no results: ->>> response.css("noelement").get() +.. code-block:: pycon + + >>> response.css("noelement").get() There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail @@ -300,14 +320,16 @@ Besides the :meth:`~scrapy.selector.SelectorList.getall` and the :meth:`~scrapy.selector.SelectorList.re` method to extract using :doc:`regular expressions `: ->>> response.css('title::text').re(r'Quotes.*') -['Quotes to Scrape'] ->>> response.css('title::text').re(r'Q\w+') -['Quotes'] ->>> response.css('title::text').re(r'(\w+) to (\w+)') -['Quotes', 'Scrape'] +.. code-block:: pycon -In order to find the proper CSS selectors to use, you might find useful opening + >>> response.css("title::text").re(r"Quotes.*") + ['Quotes to Scrape'] + >>> response.css("title::text").re(r"Q\w+") + ['Quotes'] + >>> response.css("title::text").re(r"(\w+) to (\w+)") + ['Quotes', 'Scrape'] + +In order to find the proper CSS selectors to use, you might find it useful to open the response page from the shell in your web browser using ``view(response)``. You can use your browser's developer tools to inspect the HTML and come up with a selector (see :ref:`topics-developer-tools`). @@ -323,10 +345,12 @@ XPath: a brief intro Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions: ->>> response.xpath('//title') -[] ->>> response.xpath('//title/text()').get() -'Quotes to Scrape' +.. code-block:: pycon + + >>> response.xpath("//title") + [] + >>> response.xpath("//title/text()").get() + 'Quotes to Scrape' XPath expressions are very powerful, and are the foundation of Scrapy Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You @@ -379,37 +403,45 @@ like this: Let's open up scrapy shell and play a bit to find out how to extract the data we want:: - $ scrapy shell 'https://quotes.toscrape.com' + scrapy shell 'https://quotes.toscrape.com' We get a list of selectors for the quote HTML elements with: ->>> response.css("div.quote") -[, - , - ...] +.. code-block:: pycon + + >>> response.css("div.quote") + [, + , + ...] Each of the selectors returned by the query above allows us to run further queries over their sub-elements. Let's assign the first selector to a variable, so that we can run our CSS selectors directly on a particular quote: ->>> quote = response.css("div.quote")[0] +.. code-block:: pycon + + >>> quote = response.css("div.quote")[0] Now, let's extract ``text``, ``author`` and the ``tags`` from that quote using the ``quote`` object we just created: ->>> text = quote.css("span.text::text").get() ->>> text -'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' ->>> author = quote.css("small.author::text").get() ->>> author -'Albert Einstein' +.. code-block:: pycon + + >>> text = quote.css("span.text::text").get() + >>> text + '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' + >>> author = quote.css("small.author::text").get() + >>> author + 'Albert Einstein' Given that the tags are a list of strings, we can use the ``.getall()`` method to get all of them: ->>> tags = quote.css("div.tags a.tag::text").getall() ->>> tags -['change', 'deep-thoughts', 'thinking', 'world'] +.. code-block:: pycon + + >>> tags = quote.css("div.tags a.tag::text").getall() + >>> tags + ['change', 'deep-thoughts', 'thinking', 'world'] .. invisible-code-block: python @@ -418,14 +450,17 @@ to get all of them: Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary: ->>> for quote in response.css("div.quote"): -... text = quote.css("span.text::text").get() -... author = quote.css("small.author::text").get() -... tags = quote.css("div.tags a.tag::text").getall() -... print(dict(text=text, author=author, tags=tags)) -{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']} -{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']} -... +.. code-block:: pycon + + >>> for quote in response.css("div.quote"): + ... text = quote.css("span.text::text").get() + ... author = quote.css("small.author::text").get() + ... tags = quote.css("div.tags a.tag::text").getall() + ... print(dict(text=text, author=author, tags=tags)) + ... + {'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']} + {'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']} + ... Extracting data in our spider ----------------------------- @@ -436,7 +471,9 @@ extraction logic above into our spider. A Scrapy spider typically generates many dictionaries containing the data extracted from the page. To do that, we use the ``yield`` Python keyword -in the callback, as you can see below:: +in the callback, as you can see below: + +.. code-block:: python import scrapy @@ -444,16 +481,16 @@ in the callback, as you can see below:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', - 'https://quotes.toscrape.com/page/2/', + "https://quotes.toscrape.com/page/1/", + "https://quotes.toscrape.com/page/2/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } If you run this spider, it will output the extracted data with the log:: @@ -482,7 +519,7 @@ to append new content to any existing file. However, appending to a JSON file makes the file contents invalid JSON. When appending to a file, consider using a different serialization format, such as `JSON Lines`_:: - scrapy crawl quotes -o quotes.jl + scrapy crawl quotes -o quotes.jsonl The `JSON Lines`_ format is useful because it's stream-like, you can easily append new records to it. It doesn't have the same problem of JSON when you run @@ -531,17 +568,23 @@ This gets the anchor element, but we want the attribute ``href``. For that, Scrapy supports a CSS extension that lets you select the attribute contents, like this: ->>> response.css('li.next a::attr(href)').get() -'/page/2/' +.. code-block:: pycon + + >>> response.css("li.next a::attr(href)").get() + '/page/2/' There is also an ``attrib`` property available (see :ref:`selecting-attributes` for more): ->>> response.css('li.next a').attrib['href'] -'/page/2/' +.. code-block:: pycon + + >>> response.css("li.next a").attrib["href"] + '/page/2/' Let's see now our spider modified to recursively follow the link to the next -page, extracting data from it:: +page, extracting data from it: + +.. code-block:: python import scrapy @@ -549,18 +592,18 @@ page, extracting data from it:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', + "https://quotes.toscrape.com/page/1/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) @@ -592,7 +635,9 @@ A shortcut for creating Requests -------------------------------- As a shortcut for creating Request objects you can use -:meth:`response.follow `:: +:meth:`response.follow `: + +.. code-block:: python import scrapy @@ -600,18 +645,18 @@ As a shortcut for creating Request objects you can use class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', + "https://quotes.toscrape.com/page/1/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('span small::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("span small::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: yield response.follow(next_page, callback=self.parse) @@ -619,58 +664,72 @@ Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no need to call urljoin. Note that ``response.follow`` just returns a Request instance; you still have to yield this Request. -You can also pass a selector to ``response.follow`` instead of a string; -this selector should extract necessary attributes:: +.. skip: start - for href in response.css('ul.pager a::attr(href)'): +You can also pass a selector to ``response.follow`` instead of a string; +this selector should extract necessary attributes: + +.. code-block:: python + + for href in response.css("ul.pager a::attr(href)"): yield response.follow(href, callback=self.parse) For ```` elements there is a shortcut: ``response.follow`` uses their href -attribute automatically. So the code can be shortened further:: +attribute automatically. So the code can be shortened further: - for a in response.css('ul.pager a'): +.. code-block:: python + + for a in response.css("ul.pager a"): yield response.follow(a, callback=self.parse) To create multiple requests from an iterable, you can use -:meth:`response.follow_all ` instead:: +:meth:`response.follow_all ` instead: - anchors = response.css('ul.pager a') +.. code-block:: python + + anchors = response.css("ul.pager a") yield from response.follow_all(anchors, callback=self.parse) -or, shortening it further:: +or, shortening it further: - yield from response.follow_all(css='ul.pager a', callback=self.parse) +.. code-block:: python + + yield from response.follow_all(css="ul.pager a", callback=self.parse) + +.. skip: end More examples and patterns -------------------------- Here is another spider that illustrates callbacks and following links, -this time for scraping author information:: +this time for scraping author information: + +.. code-block:: python import scrapy class AuthorSpider(scrapy.Spider): - name = 'author' + name = "author" - start_urls = ['https://quotes.toscrape.com/'] + start_urls = ["https://quotes.toscrape.com/"] def parse(self, response): - author_page_links = response.css('.author + a') + author_page_links = response.css(".author + a") yield from response.follow_all(author_page_links, self.parse_author) - pagination_links = response.css('li.next a') + pagination_links = response.css("li.next a") yield from response.follow_all(pagination_links, self.parse) def parse_author(self, response): def extract_with_css(query): - return response.css(query).get(default='').strip() + return response.css(query).get(default="").strip() yield { - 'name': extract_with_css('h3.author-title::text'), - 'birthdate': extract_with_css('.author-born-date::text'), - 'bio': extract_with_css('.author-description::text'), + "name": extract_with_css("h3.author-title::text"), + "birthdate": extract_with_css(".author-born-date::text"), + "bio": extract_with_css(".author-description::text"), } This spider will start from the main page, it will follow all the links to the @@ -718,7 +777,9 @@ spider attributes by default. In this example, the value provided for the ``tag`` argument will be available via ``self.tag``. You can use this to make your spider fetch only quotes -with a specific tag, building the URL based on the argument:: +with a specific tag, building the URL based on the argument: + +.. code-block:: python import scrapy @@ -727,20 +788,20 @@ with a specific tag, building the URL based on the argument:: name = "quotes" def start_requests(self): - url = 'https://quotes.toscrape.com/' - tag = getattr(self, 'tag', None) + url = "https://quotes.toscrape.com/" + tag = getattr(self, "tag", None) if tag is not None: - url = url + 'tag/' + tag + url = url + "tag/" + tag yield scrapy.Request(url, self.parse) def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: yield response.follow(next_page, self.parse) diff --git a/docs/news.rst b/docs/news.rst index 5d92067b5..c7ad11862 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,673 @@ Release notes ============= +.. _release-2.9.0: + +Scrapy 2.9.0 (2023-05-08) +------------------------- + +Highlights: + +- Per-domain download settings. +- Compatibility with new cryptography_ and new parsel_. +- JMESPath selectors from the new parsel_. +- Bug fixes. + +Deprecations +~~~~~~~~~~~~ + +- :class:`scrapy.extensions.feedexport._FeedSlot` is renamed to + :class:`scrapy.extensions.feedexport.FeedSlot` and the old name is + deprecated. (:issue:`5876`) + +New features +~~~~~~~~~~~~ + +- Settings correponding to :setting:`DOWNLOAD_DELAY`, + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis + via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) + +- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors + available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) + +- Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed` + signals. (:issue:`5876`) + +- Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a + curl command from a :class:`~scrapy.Request` object. (:issue:`5892`) + +- Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be + :class:`pathlib.Path` instances. (:issue:`5801`) + +Bug fixes +~~~~~~~~~ + +- Fixed a warning with Parsel 1.8.1+. (:issue:`5903`, :issue:`5918`) + +- Fixed an error when using feed postprocessing with S3 storage. + (:issue:`5500`, :issue:`5581`) + +- Added the missing :meth:`scrapy.settings.BaseSettings.setdefault` method. + (:issue:`5811`, :issue:`5821`) + +- Fixed an error when using cryptography_ 40.0.0+ and + :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` is enabled. + (:issue:`5857`, :issue:`5858`) + +- The checksums returned by :class:`~scrapy.pipelines.files.FilesPipeline` + for files on Google Cloud Storage are no longer Base64-encoded. + (:issue:`5874`, :issue:`5891`) + +- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed + string values for the curl ``--data-raw`` argument, which are produced by + browsers for data that includes certain symbols. (:issue:`5899`, + :issue:`5901`) + +- The :command:`parse` command now also works with async generator callbacks. + (:issue:`5819`, :issue:`5824`) + +- The :command:`genspider` command now properly works with HTTPS URLs. + (:issue:`3553`, :issue:`5808`) + +- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) + +- :class:`LinkExtractor ` + now skips certain malformed URLs instead of raising an exception. + (:issue:`5881`) + +- :func:`scrapy.utils.python.get_func_args` now supports more types of + callables. (:issue:`5872`, :issue:`5885`) + +- Fixed an error when processing non-UTF8 values of ``Content-Type`` headers. + (:issue:`5914`, :issue:`5917`) + +- Fixed an error breaking user handling of send failures in + :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) + +Documentation +~~~~~~~~~~~~~ + +- Expanded contributing docs. (:issue:`5109`, :issue:`5851`) + +- Added blacken-docs_ to pre-commit and reformatted the docs with it. + (:issue:`5813`, :issue:`5816`) + +- Fixed a JS issue. (:issue:`5875`, :issue:`5877`) + +- Fixed ``make htmlview``. (:issue:`5878`, :issue:`5879`) + +- Fixed typos and other small errors. (:issue:`5827`, :issue:`5839`, + :issue:`5883`, :issue:`5890`, :issue:`5895`, :issue:`5904`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`) + +- Tests for most of the examples in the docs are now run as a part of CI, + found problems were fixed. (:issue:`5816`, :issue:`5826`, :issue:`5919`) + +- Removed usage of deprecated Python classes. (:issue:`5849`) + +- Silenced ``include-ignored`` warnings from coverage. (:issue:`5820`) + +- Fixed a random failure of the ``test_feedexport.test_batch_path_differ`` + test. (:issue:`5855`, :issue:`5898`) + +- Updated docstrings to match output produced by parsel_ 1.8.1 so that they + don't cause test failures. (:issue:`5902`, :issue:`5919`) + +- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`, + :issue:`5908`) + +.. _blacken-docs: https://github.com/adamchainz/blacken-docs + +.. _release-2.8.0: + +Scrapy 2.8.0 (2023-02-02) +------------------------- + +This is a maintenance release, with minor features, bug fixes, and cleanups. + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``scrapy.utils.gz.read1`` function, deprecated in Scrapy 2.0, has now + been removed. Use the :meth:`~io.BufferedIOBase.read1` method of + :class:`~gzip.GzipFile` instead. + (:issue:`5719`) + +- The ``scrapy.utils.python.to_native_str`` function, deprecated in Scrapy + 2.0, has now been removed. Use :func:`scrapy.utils.python.to_unicode` + instead. + (:issue:`5719`) + +- The ``scrapy.utils.python.MutableChain.next`` method, deprecated in Scrapy + 2.0, has now been removed. Use + :meth:`~scrapy.utils.python.MutableChain.__next__` instead. + (:issue:`5719`) + +- The ``scrapy.linkextractors.FilteringLinkExtractor`` class, deprecated + in Scrapy 2.0, has now been removed. Use + :class:`LinkExtractor ` + instead. + (:issue:`5720`) + +- Support for using environment variables prefixed with ``SCRAPY_`` to + override settings, deprecated in Scrapy 2.0, has now been removed. + (:issue:`5724`) + +- Support for the ``noconnect`` query string argument in proxy URLs, + deprecated in Scrapy 2.0, has now been removed. We expect proxies that used + to need it to work fine without it. + (:issue:`5731`) + +- The ``scrapy.utils.python.retry_on_eintr`` function, deprecated in Scrapy + 2.3, has now been removed. + (:issue:`5719`) + +- The ``scrapy.utils.python.WeakKeyCache`` class, deprecated in Scrapy 2.4, + has now been removed. + (:issue:`5719`) + + +Deprecations +~~~~~~~~~~~~ + +- :exc:`scrapy.pipelines.images.NoimagesDrop` is now deprecated. + (:issue:`5368`, :issue:`5489`) + +- :meth:`ImagesPipeline.convert_image + ` must now accept a + ``response_body`` parameter. + (:issue:`3055`, :issue:`3689`, :issue:`4753`) + + +New features +~~~~~~~~~~~~ + +- Applied black_ coding style to files generated with the + :command:`genspider` and :command:`startproject` commands. + (:issue:`5809`, :issue:`5814`) + + .. _black: https://black.readthedocs.io/en/stable/ + +- :setting:`FEED_EXPORT_ENCODING` is now set to ``"utf-8"`` in the + ``settings.py`` file that the :command:`startproject` command generates. + With this value, JSON exports won’t force the use of escape sequences for + non-ASCII characters. + (:issue:`5797`, :issue:`5800`) + +- The :class:`~scrapy.extensions.memusage.MemoryUsage` extension now logs the + peak memory usage during checks, and the binary unit MiB is now used to + avoid confusion. + (:issue:`5717`, :issue:`5722`, :issue:`5727`) + +- The ``callback`` parameter of :class:`~scrapy.http.Request` can now be set + to :func:`scrapy.http.request.NO_CALLBACK`, to distinguish it from + ``None``, as the latter indicates that the default spider callback + (:meth:`~scrapy.Spider.parse`) is to be used. + (:issue:`5798`) + + +Bug fixes +~~~~~~~~~ + +- Enabled unsafe legacy SSL renegotiation to fix access to some outdated + websites. + (:issue:`5491`, :issue:`5790`) + +- Fixed STARTTLS-based email delivery not working with Twisted 21.2.0 and + better. + (:issue:`5386`, :issue:`5406`) + +- Fixed the :meth:`finish_exporting` method of :ref:`item exporters + ` not being called for empty files. + (:issue:`5537`, :issue:`5758`) + +- Fixed HTTP/2 responses getting only the last value for a header when + multiple headers with the same name are received. + (:issue:`5777`) + +- Fixed an exception raised by the :command:`shell` command on some cases + when :ref:`using asyncio `. + (:issue:`5740`, :issue:`5742`, :issue:`5748`, :issue:`5759`, :issue:`5760`, + :issue:`5771`) + +- When using :class:`~scrapy.spiders.CrawlSpider`, callback keyword arguments + (``cb_kwargs``) added to a request in the ``process_request`` callback of a + :class:`~scrapy.spiders.Rule` will no longer be ignored. + (:issue:`5699`) + +- The :ref:`images pipeline ` no longer re-encodes JPEG + files. + (:issue:`3055`, :issue:`3689`, :issue:`4753`) + +- Fixed the handling of transparent WebP images by the :ref:`images pipeline + `. + (:issue:`3072`, :issue:`5766`, :issue:`5767`) + +- :func:`scrapy.shell.inspect_response` no longer inhibits ``SIGINT`` + (Ctrl+C). + (:issue:`2918`) + +- :class:`LinkExtractor ` + with ``unique=False`` no longer filters out links that have identical URL + *and* text. + (:issue:`3798`, :issue:`3799`, :issue:`4695`, :issue:`5458`) + +- :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` now + ignores URL protocols that do not support ``robots.txt`` (``data://``, + ``file://``). + (:issue:`5807`) + +- Silenced the ``filelock`` debug log messages introduced in Scrapy 2.6. + (:issue:`5753`, :issue:`5754`) + +- Fixed the output of ``scrapy -h`` showing an unintended ``**commands**`` + line. + (:issue:`5709`, :issue:`5711`, :issue:`5712`) + +- Made the active project indication in the output of :ref:`commands + ` more clear. + (:issue:`5715`) + + +Documentation +~~~~~~~~~~~~~ + +- Documented how to :ref:`debug spiders from Visual Studio Code + `. + (:issue:`5721`) + +- Documented how :setting:`DOWNLOAD_DELAY` affects per-domain concurrency. + (:issue:`5083`, :issue:`5540`) + +- Improved consistency. + (:issue:`5761`) + +- Fixed typos. + (:issue:`5714`, :issue:`5744`, :issue:`5764`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Applied :ref:`black coding style `, sorted import statements, + and introduced :ref:`pre-commit `. + (:issue:`4654`, :issue:`4658`, :issue:`5734`, :issue:`5737`, :issue:`5806`, + :issue:`5810`) + +- Switched from :mod:`os.path` to :mod:`pathlib`. + (:issue:`4916`, :issue:`4497`, :issue:`5682`) + +- Addressed many issues reported by Pylint. + (:issue:`5677`) + +- Improved code readability. + (:issue:`5736`) + +- Improved package metadata. + (:issue:`5768`) + +- Removed direct invocations of ``setup.py``. + (:issue:`5774`, :issue:`5776`) + +- Removed unnecessary :class:`~collections.OrderedDict` usages. + (:issue:`5795`) + +- Removed unnecessary ``__str__`` definitions. + (:issue:`5150`) + +- Removed obsolete code and comments. + (:issue:`5725`, :issue:`5729`, :issue:`5730`, :issue:`5732`) + +- Fixed test and CI issues. + (:issue:`5749`, :issue:`5750`, :issue:`5756`, :issue:`5762`, :issue:`5765`, + :issue:`5780`, :issue:`5781`, :issue:`5782`, :issue:`5783`, :issue:`5785`, + :issue:`5786`) + + +.. _release-2.7.1: + +Scrapy 2.7.1 (2022-11-02) +------------------------- + +New features +~~~~~~~~~~~~ + +- Relaxed the restriction introduced in 2.6.2 so that the + ``Proxy-Authorization`` header can again be set explicitly, as long as the + proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and + for as long as that proxy URL remains the same; this restores compatibility + with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`). + +Bug fixes +~~~~~~~~~ + +- Using ``-O``/``--overwrite-output`` and ``-t``/``--output-format`` options + together now produces an error instead of ignoring the former option + (:issue:`5516`, :issue:`5605`). + +- Replaced deprecated :mod:`asyncio` APIs that implicitly use the current + event loop with code that explicitly requests a loop from the event loop + policy (:issue:`5685`, :issue:`5689`). + +- Fixed uses of deprecated Scrapy APIs in Scrapy itself (:issue:`5588`, + :issue:`5589`). + +- Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`). + +- Improved code that checks if generators return values, so that it no longer + fails on decorated methods and partial methods (:issue:`5323`, + :issue:`5592`, :issue:`5599`, :issue:`5691`). + +Documentation +~~~~~~~~~~~~~ + +- Upgraded the Code of Conduct to Contributor Covenant v2.1 (:issue:`5698`). + +- Fixed typos (:issue:`5681`, :issue:`5694`). + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Re-enabled some erroneously disabled flake8 checks (:issue:`5688`). + +- Ignored harmless deprecation warnings from :mod:`typing` in tests + (:issue:`5686`, :issue:`5697`). + +- Modernized our CI configuration (:issue:`5695`, :issue:`5696`). + + +.. _release-2.7.0: + +Scrapy 2.7.0 (2022-10-17) +----------------------------- + +Highlights: + +- Added Python 3.11 support, dropped Python 3.6 support +- Improved support for :ref:`asynchronous callbacks ` +- :ref:`Asyncio support ` is enabled by default on new + projects +- Output names of item fields can now be arbitrary strings +- Centralized :ref:`request fingerprinting ` + configuration is now possible + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +Python 3.7 or greater is now required; support for Python 3.6 has been dropped. +Support for the upcoming Python 3.11 has been added. + +The minimum required version of some dependencies has changed as well: + +- lxml_: 3.5.0 → 4.3.0 + +- Pillow_ (:ref:`images pipeline `): 4.0.0 → 7.1.0 + +- zope.interface_: 5.0.0 → 5.1.0 + +(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`, +:issue:`5670`, :issue:`5678`) + + +Deprecations +~~~~~~~~~~~~ + +- :meth:`ImagesPipeline.thumb_path + ` must now accept an + ``item`` parameter (:issue:`5504`, :issue:`5508`). + +- The ``scrapy.downloadermiddlewares.decompression`` module is now + deprecated (:issue:`5546`, :issue:`5547`). + + +New features +~~~~~~~~~~~~ + +- The + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` + method of :ref:`spider middlewares ` can now be + defined as an :term:`asynchronous generator` (:issue:`4978`). + +- The output of :class:`~scrapy.Request` callbacks defined as + :ref:`coroutines ` is now processed asynchronously + (:issue:`4978`). + +- :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous + callbacks ` (:issue:`5657`). + +- New projects created with the :command:`startproject` command have + :ref:`asyncio support ` enabled by default (:issue:`5590`, + :issue:`5679`). + +- The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a + dictionary to customize the output name of item fields, lifting the + restriction that required output names to be valid Python identifiers, e.g. + preventing them to have whitespace (:issue:`1008`, :issue:`3266`, + :issue:`3696`). + +- You can now customize :ref:`request fingerprinting ` + through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of + having to change it on every Scrapy component that relies on request + fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`, + :issue:`4524`). + +- ``jsonl`` is now supported and encouraged as a file extension for `JSON + Lines`_ files (:issue:`4848`). + + .. _JSON Lines: https://jsonlines.org/ + +- :meth:`ImagesPipeline.thumb_path + ` now receives the + source :ref:`item ` (:issue:`5504`, :issue:`5508`). + + +Bug fixes +~~~~~~~~~ + +- When using Google Cloud Storage with a :ref:`media pipeline + `, :setting:`FILES_EXPIRES` now also works when + :setting:`FILES_STORE` does not point at the root of your Google Cloud + Storage bucket (:issue:`5317`, :issue:`5318`). + +- The :command:`parse` command now supports :ref:`asynchronous callbacks + ` (:issue:`5424`, :issue:`5577`). + +- When using the :command:`parse` command with a URL for which there is no + available spider, an exception is no longer raised (:issue:`3264`, + :issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`). + +- :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte + order mark`_ when determining the text encoding of the response body, + following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`). + + .. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark + .. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding + +- MIME sniffing takes the response body into account in FTP and HTTP/1.0 + requests, as well as in cached requests (:issue:`4873`). + +- MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag + is missing (:issue:`4873`). + +- An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value + that does not match the asyncio event loop actually installed + (:issue:`5529`). + +- Fixed :meth:`Headers.getlist ` + returning only the last header (:issue:`5515`, :issue:`5526`). + +- Fixed :class:`LinkExtractor + ` not ignoring the + ``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`, + :issue:`4066`) + + +Documentation +~~~~~~~~~~~~~ + +- Clarified the return type of :meth:`Spider.parse ` + (:issue:`5602`, :issue:`5608`). + +- To enable + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + to do `brotli compression`_, installing brotli_ is now recommended instead + of installing brotlipy_, as the former provides a more recent version of + brotli. + + .. _brotli: https://github.com/google/brotli + .. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt + +- :ref:`Signal documentation ` now mentions :ref:`coroutine + support ` and uses it in code examples (:issue:`4852`, + :issue:`5358`). + +- :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_ + (:issue:`3582`, :issue:`5432`). + + .. _Common Crawl: https://commoncrawl.org/ + .. _Google cache: http://www.googleguide.com/cached_pages.html + +- The new :ref:`topics-components` topic covers enforcing requirements on + Scrapy components, like :ref:`downloader middlewares + `, :ref:`extensions `, + :ref:`item pipelines `, :ref:`spider middlewares + `, and more; :ref:`enforce-asyncio-requirement` + has also been added (:issue:`4978`). + +- :ref:`topics-settings` now indicates that setting values must be + :ref:`picklable ` (:issue:`5607`, :issue:`5629`). + +- Removed outdated documentation (:issue:`5446`, :issue:`5373`, + :issue:`5369`, :issue:`5370`, :issue:`5554`). + +- Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`, + :issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`). + +- Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`, + :issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`). + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added a continuous integration job to run `twine check`_ (:issue:`5655`, + :issue:`5656`). + + .. _twine check: https://twine.readthedocs.io/en/stable/#twine-check + +- Addressed test issues and warnings (:issue:`5560`, :issue:`5561`, + :issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`, + :issue:`5671`, :issue:`5675`). + +- Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`, + :issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`). + +- Applied minor code improvements (:issue:`5661`). + + +.. _release-2.6.3: + +Scrapy 2.6.3 (2022-09-27) +------------------------- + +- Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3 + (:issue:`5634`, :issue:`5635`, :issue:`5636`). + +- Upgraded the minimum versions of the following dependencies: + + - cryptography_: 2.0 → 3.3 + + - pyOpenSSL_: 16.2.0 → 21.0.0 + + - service_identity_: 16.0.0 → 18.1.0 + + - Twisted_: 17.9.0 → 18.9.0 + + - zope.interface_: 4.1.3 → 5.0.0 + + (:issue:`5621`, :issue:`5632`) + +- Fixes test and documentation issues (:issue:`5612`, :issue:`5617`, + :issue:`5631`). + + +.. _release-2.6.2: + +Scrapy 2.6.2 (2022-07-25) +------------------------- + +**Security bug fix:** + +- When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + processes a request with :reqmeta:`proxy` metadata, and that + :reqmeta:`proxy` metadata includes proxy credentials, + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets + the ``Proxy-Authorization`` header, but only if that header is not already + set. + + There are third-party proxy-rotation downloader middlewares that set + different :reqmeta:`proxy` metadata every time they process a request. + + Because of request retries and redirects, the same request can be processed + by downloader middlewares more than once, including both + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and + any third-party proxy-rotation downloader middleware. + + These third-party proxy-rotation downloader middlewares could change the + :reqmeta:`proxy` metadata of a request to a new value, but fail to remove + the ``Proxy-Authorization`` header from the previous value of the + :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent + to a different proxy. + + To prevent the unintended leaking of proxy credentials, the behavior of + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now + as follows when processing a request: + + - If the request being processed defines :reqmeta:`proxy` metadata that + includes credentials, the ``Proxy-Authorization`` header is always + updated to feature those credentials. + + - If the request being processed defines :reqmeta:`proxy` metadata + without credentials, the ``Proxy-Authorization`` header is removed + *unless* it was originally defined for the same proxy URL. + + To remove proxy credentials while keeping the same proxy URL, remove + the ``Proxy-Authorization`` header. + + - If the request has no :reqmeta:`proxy` metadata, or that metadata is a + falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is + removed. + + It is no longer possible to set a proxy URL through the + :reqmeta:`proxy` metadata but set the credentials through the + ``Proxy-Authorization`` header. Set proxy credentials through the + :reqmeta:`proxy` metadata instead. + +Also fixes the following regressions introduced in 2.6.0: + +- :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple + spiders (:issue:`5435`, :issue:`5436`) + +- Installing a Twisted reactor before Scrapy does (e.g. importing + :mod:`twisted.internet.reactor` somewhere at the module level) no longer + prevents Scrapy from starting, as long as a different reactor is not + specified in :setting:`TWISTED_REACTOR` (:issue:`5525`, :issue:`5528`) + +- Fixed an exception that was being logged after the spider finished under + certain conditions (:issue:`5437`, :issue:`5440`) + +- The ``--output``/``-o`` command-line parameter supports again a value + starting with a hyphen (:issue:`5444`, :issue:`5445`) + +- The ``scrapy parse -h`` command no longer throws an error (:issue:`5481`, + :issue:`5482`) + + .. _release-2.6.1: Scrapy 2.6.1 (2022-03-01) @@ -113,6 +780,9 @@ Backward-incompatible changes meet expectations, :exc:`TypeError` is now raised at startup time. Before, other exceptions would be raised at run time. (:issue:`3559`) +- The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects + is now named ``encoding``, in line with all other fields (:issue:`5130`) + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ @@ -1643,7 +2313,7 @@ New features :issue:`4370`) * A new ``keep_fragments`` parameter of - :func:`scrapy.utils.request.request_fingerprint` allows to generate + ``scrapy.utils.request.request_fingerprint`` allows to generate different fingerprints for requests with different fragments in their URL (:issue:`4104`) @@ -1897,6 +2567,59 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.3: + +Scrapy 1.8.3 (2022-07-25) +------------------------- + +**Security bug fix:** + +- When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + processes a request with :reqmeta:`proxy` metadata, and that + :reqmeta:`proxy` metadata includes proxy credentials, + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets + the ``Proxy-Authorization`` header, but only if that header is not already + set. + + There are third-party proxy-rotation downloader middlewares that set + different :reqmeta:`proxy` metadata every time they process a request. + + Because of request retries and redirects, the same request can be processed + by downloader middlewares more than once, including both + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and + any third-party proxy-rotation downloader middleware. + + These third-party proxy-rotation downloader middlewares could change the + :reqmeta:`proxy` metadata of a request to a new value, but fail to remove + the ``Proxy-Authorization`` header from the previous value of the + :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent + to a different proxy. + + To prevent the unintended leaking of proxy credentials, the behavior of + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now + as follows when processing a request: + + - If the request being processed defines :reqmeta:`proxy` metadata that + includes credentials, the ``Proxy-Authorization`` header is always + updated to feature those credentials. + + - If the request being processed defines :reqmeta:`proxy` metadata + without credentials, the ``Proxy-Authorization`` header is removed + *unless* it was originally defined for the same proxy URL. + + To remove proxy credentials while keeping the same proxy URL, remove + the ``Proxy-Authorization`` header. + + - If the request has no :reqmeta:`proxy` metadata, or that metadata is a + falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is + removed. + + It is no longer possible to set a proxy URL through the + :reqmeta:`proxy` metadata but set the credentials through the + ``Proxy-Authorization`` header. Set proxy credentials through the + :reqmeta:`proxy` metadata instead. + + .. _release-1.8.2: Scrapy 1.8.2 (2022-03-01) @@ -2022,11 +2745,13 @@ Backward-incompatible changes * :class:`~scrapy.loader.ItemLoader` now turns the values of its input item into lists: - >>> item = MyItem() - >>> item['field'] = 'value1' - >>> loader = ItemLoader(item=item) - >>> item['field'] - ['value1'] + .. code-block:: pycon + + >>> item = MyItem() + >>> item["field"] = "value1" + >>> loader = ItemLoader(item=item) + >>> item["field"] + ['value1'] This is needed to allow adding values to existing fields (``loader.add_value('field', 'value2')``). @@ -2985,7 +3710,7 @@ New Features ~~~~~~~~~~~~ - Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`) -- Support `brotli`_-compressed content; requires optional `brotlipy`_ +- Support `brotli-compressed`_ content; requires optional `brotlipy`_ (:issue:`2535`) - New :ref:`response.follow ` shortcut for creating requests (:issue:`1940`) @@ -3022,7 +3747,7 @@ New Features - ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command (:issue:`2740`) -.. _brotli: https://github.com/google/brotli +.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://github.com/python-hyper/brotlipy/ Bug fixes @@ -3604,8 +4329,6 @@ Relocations + Note: telnet is not enabled on Python 3 (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595) -.. _parsel: https://github.com/scrapy/parsel - Bugfixes ~~~~~~~~ @@ -4305,7 +5028,7 @@ Scrapy 0.22.1 (released 2014-02-08) - BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`) - BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`) - Fix tests for Travis-CI build (:commit:`76c7e20`) -- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`) +- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`) - RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`) - Updated the tutorial crawl output with latest output. (:commit:`8da65de`) - Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`) @@ -4330,7 +5053,7 @@ Enhancements - [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`) To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage`` - Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`) -- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`) +- Add a middleware to crawl ajax crawlable pages as defined by google (:issue:`343`) - Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`) - Selectors register EXSLT namespaces by default (:issue:`472`) - Unify item loaders similar to selectors renaming (:issue:`461`) @@ -4510,7 +5233,7 @@ Scrapy 0.18.0 (released 2013-08-09) ----------------------------------- - Lot of improvements to testsuite run using Tox, including a way to test on pypi -- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`) +- Handle GET parameters for AJAX crawlable urls (:commit:`3fe2a32`) - Use lxml recover option to parse sitemaps (:issue:`347`) - Bugfix cookie merging by hostname and not by netloc (:issue:`352`) - Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`) @@ -4544,8 +5267,8 @@ Scrapy 0.18.0 (released 2013-08-09) - Added ``--pdb`` option to ``scrapy`` command line tool - Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. - Several improvements to spider contracts -- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections, -- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 +- New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections, +- MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62 - added from_crawler method to spiders - added system tests with mock server - more improvements to macOS compatibility (thanks Alex Cepoi) @@ -4687,7 +5410,7 @@ Scrapy changes: - promoted :ref:`topics-djangoitem` to main contrib - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`) - downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method -- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module +- replaced memory usage accounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module - removed signal: ``scrapy.mail.mail_sent`` - removed ``TRACK_REFS`` setting, now :ref:`trackrefs ` is always enabled - DBM is now the default storage backend for HTTP cache middleware @@ -4753,7 +5476,7 @@ Scrapy 0.14 New features and settings ~~~~~~~~~~~~~~~~~~~~~~~~~ -- Support for `AJAX crawleable urls`_ +- Support for `AJAX crawlable urls`_ - New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`) - added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``) - Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`) @@ -5013,7 +5736,7 @@ Backward-incompatible changes - Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`) - Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`) - Refactored HTTP Cache middleware -- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` ) +- HTTP Cache middleware has been heavily refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` ) - Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120) - Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121) - Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`) @@ -5024,7 +5747,7 @@ Scrapy 0.7 First release of Scrapy. -.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 +.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ @@ -5035,6 +5758,7 @@ First release of Scrapy. .. _LevelDB: https://github.com/google/leveldb .. _lxml: https://lxml.de/ .. _marshal: https://docs.python.org/2/library/marshal.html +.. _parsel: https://github.com/scrapy/parsel .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr diff --git a/docs/requirements.txt b/docs/requirements.txt index a0930ba1e..9f9aef711 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -Sphinx>=3.0 -sphinx-hoverxref>=0.2b1 -sphinx-notfound-page>=0.4 -sphinx-rtd-theme>=0.5.2 \ No newline at end of file +sphinx==5.0.2 +sphinx-hoverxref==1.1.1 +sphinx-notfound-page==0.8 +sphinx-rtd-theme==1.0.0 diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 900b19c7a..268344879 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares :class:`scrapy.Spider` subclass and a :class:`scrapy.settings.Settings` object. + .. attribute:: request_fingerprinter + + The request fingerprint builder of this crawler. + + This is used from extensions and middlewares to build short, unique + identifiers for requests. See :ref:`request-fingerprints`. + .. attribute:: settings The settings manager of this crawler. @@ -125,16 +132,14 @@ Settings API precedence over lesser ones when setting and retrieving values in the :class:`~scrapy.settings.Settings` class. - .. highlight:: python - - :: + .. code-block:: python SETTINGS_PRIORITIES = { - 'default': 0, - 'command': 10, - 'project': 20, - 'spider': 30, - 'cmdline': 40, + "default": 0, + "command": 10, + "project": 20, + "spider": 30, + "cmdline": 40, } For a detailed explanation on each settings sources, see: diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 3a6941a2c..7713b1af1 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -96,3 +96,29 @@ Futures. Scrapy provides two helpers for this: down to Scrapy 2.0 (earlier versions do not support :mod:`asyncio`), you can copy the implementation of these functions into your own code. + + +.. _enforce-asyncio-requirement: + +Enforcing asyncio as a requirement +================================== + +If you are writing a :ref:`component ` that requires asyncio +to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to +:ref:`enforce it as a requirement `. For +example: + +.. code-block:: python + + from scrapy.utils.reactor import is_asyncio_reactor_installed + + + class MyComponent: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise ValueError( + f"{MyComponent.__qualname__} requires the asyncio Twisted " + f"reactor. Make sure you have it configured in the " + f"TWISTED_REACTOR setting. See the asyncio documentation " + f"of Scrapy for more information." + ) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 63b60312e..8be89feb2 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -48,9 +48,11 @@ Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQ It works best during single-domain crawl. It does not work well with crawling many different domains in parallel -To apply the recommended priority queue use:: +To apply the recommended priority queue use: - SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue' +.. code-block:: python + + SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue" .. _broad-crawls-concurrency: @@ -68,10 +70,12 @@ IP (:setting:`CONCURRENT_REQUESTS_PER_IP`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much -to increase it will depend on how much CPU and memory you crawler will have +to increase it will depend on how much CPU and memory your crawler will have available. -A good starting point is ``100``:: +A good starting point is ``100``: + +.. code-block:: python CONCURRENT_REQUESTS = 100 @@ -92,7 +96,9 @@ hitting DNS resolver timeouts. Possible solution to increase the number of threads handling DNS queries. The DNS queue will be processed faster speeding up establishing of connection and crawling overall. -To increase maximum thread pool size use:: +To increase maximum thread pool size use: + +.. code-block:: python REACTOR_THREADPOOL_MAXSIZE = 20 @@ -114,9 +120,11 @@ should not use ``DEBUG`` log level when preforming large broad crawls in production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. -To set the log level use:: +To set the log level use: - LOG_LEVEL = 'INFO' +.. code-block:: python + + LOG_LEVEL = "INFO" Disable cookies =============== @@ -126,7 +134,9 @@ doing broad crawls (search engine crawlers ignore them), and they improve performance by saving some CPU cycles and reducing the memory footprint of your Scrapy crawler. -To disable cookies use:: +To disable cookies use: + +.. code-block:: python COOKIES_ENABLED = False @@ -138,7 +148,9 @@ when sites causes are very slow (or fail) to respond, thus causing a timeout error which gets retried many times, unnecessarily, preventing crawler capacity to be reused for other domains. -To disable retries use:: +To disable retries use: + +.. code-block:: python RETRY_ENABLED = False @@ -149,7 +161,9 @@ Unless you are crawling from a very slow connection (which shouldn't be the case for broad crawls) reduce the download timeout so that stuck requests are discarded quickly and free up capacity to process the next ones. -To reduce the download timeout use:: +To reduce the download timeout use: + +.. code-block:: python DOWNLOAD_TIMEOUT = 15 @@ -162,7 +176,9 @@ revisiting the site at a later crawl. This also help to keep the number of request constant per crawl batch, otherwise redirect loops may cause the crawler to dedicate too many resources on any specific domain. -To disable redirects use:: +To disable redirects use: + +.. code-block:: python REDIRECT_ENABLED = False @@ -179,7 +195,9 @@ Pages can indicate it in two ways: "main", "index" website pages. Scrapy handles (1) automatically; to handle (2) enable -:ref:`AjaxCrawlMiddleware `:: +:ref:`AjaxCrawlMiddleware `: + +.. code-block:: python AJAXCRAWL_ENABLED = True diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8c0b8e55f..1d37895c2 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -238,9 +238,6 @@ genspider Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. -.. note:: Even if an HTTPS URL is specified, the protocol used in - ``start_urls`` is always HTTP. This is a known issue: :issue:`3553`. - Usage example:: $ scrapy genspider -l @@ -271,11 +268,31 @@ crawl Start crawling using a spider. +Supported options: + +* ``-h, --help``: show a help message and exit + +* ``-a NAME=VALUE``: set a spider argument (may be repeated) + +* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) + +* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) + +* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` + Usage examples:: $ scrapy crawl myspider [ ... myspider starts crawling ... ] + $ scrapy crawl -o myfile:csv myspider + [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] + + $ scrapy crawl -O myfile:json myspider + [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] + + $ scrapy crawl -o myfile -t csv myspider + [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] .. command:: check @@ -597,7 +614,7 @@ Example: .. code-block:: python - COMMANDS_MODULE = 'mybot.commands' + COMMANDS_MODULE = "mybot.commands" .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html @@ -616,10 +633,11 @@ The following example adds ``my_command`` command: from setuptools import setup, find_packages - setup(name='scrapy-mymodule', - entry_points={ - 'scrapy.commands': [ - 'my_command=my_scrapy_module.commands:MyCommand', - ], - }, - ) + setup( + name="scrapy-mymodule", + entry_points={ + "scrapy.commands": [ + "my_command=my_scrapy_module.commands:MyCommand", + ], + }, + ) diff --git a/docs/topics/components.rst b/docs/topics/components.rst new file mode 100644 index 000000000..478dd9647 --- /dev/null +++ b/docs/topics/components.rst @@ -0,0 +1,86 @@ +.. _topics-components: + +========== +Components +========== + +A Scrapy component is any class whose objects are created using +:func:`scrapy.utils.misc.create_instance`. + +That includes the classes that you may assign to the following settings: + +- :setting:`DNS_RESOLVER` + +- :setting:`DOWNLOAD_HANDLERS` + +- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` + +- :setting:`DOWNLOADER_MIDDLEWARES` + +- :setting:`DUPEFILTER_CLASS` + +- :setting:`EXTENSIONS` + +- :setting:`FEED_EXPORTERS` + +- :setting:`FEED_STORAGES` + +- :setting:`ITEM_PIPELINES` + +- :setting:`SCHEDULER` + +- :setting:`SCHEDULER_DISK_QUEUE` + +- :setting:`SCHEDULER_MEMORY_QUEUE` + +- :setting:`SCHEDULER_PRIORITY_QUEUE` + +- :setting:`SPIDER_MIDDLEWARES` + +Third-party Scrapy components may also let you define additional Scrapy +components, usually configurable through :ref:`settings `, to +modify their behavior. + +.. _enforce-component-requirements: + +Enforcing component requirements +================================ + +Sometimes, your components may only be intended to work under certain +conditions. For example, they may require a minimum version of Scrapy to work as +intended, or they may require certain settings to have specific values. + +In addition to describing those conditions in the documentation of your +component, it is a good practice to raise an exception from the ``__init__`` +method of your component if those conditions are not met at run time. + +In the case of :ref:`downloader middlewares `, +:ref:`extensions `, :ref:`item pipelines +`, and :ref:`spider middlewares +`, you should raise +:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a +parameter to the exception so that it is printed in the logs, for the user to +see. For other components, feel free to raise whatever other exception feels +right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy +version mismatch, while :exc:`ValueError` may be better if the issue is the +value of a setting. + +If your requirement is a minimum Scrapy version, you may use +:attr:`scrapy.__version__` to enforce your requirement. For example: + +.. code-block:: python + + from packaging.version import parse as parse_version + + import scrapy + + + class MyComponent: + def __init__(self): + if parse_version(scrapy.__version__) < parse_version("2.7"): + raise RuntimeError( + f"{MyComponent.__qualname__} requires Scrapy 2.7 or " + f"later, which allow defining the process_spider_output " + f"method of spider middlewares as an asynchronous " + f"generator." + ) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index ef296dc9e..2d61026e9 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -11,10 +11,13 @@ integrated way of testing your spiders by the means of contracts. This allows you to test each callback of your spider by hardcoding a sample url and check various constraints for how the callback processes the response. Each contract is prefixed with an ``@`` and included in the docstring. See the -following example:: +following example: + +.. code-block:: python def parse(self, response): - """ This function parses a sample response. Some contracts are mingled + """ + This function parses a sample response. Some contracts are mingled with this docstring. @url http://www.amazon.com/s?field-keywords=selfish+gene @@ -64,11 +67,13 @@ Custom Contracts If you find you need more power than the built-in Scrapy contracts you can create and load your own contracts in the project by using the -:setting:`SPIDER_CONTRACTS` setting:: +:setting:`SPIDER_CONTRACTS` setting: + +.. code-block:: python SPIDER_CONTRACTS = { - 'myproject.contracts.ResponseCheck': 10, - 'myproject.contracts.ItemValidate': 10, + "myproject.contracts.ResponseCheck": 10, + "myproject.contracts.ItemValidate": 10, } Each contract must inherit from :class:`~scrapy.contracts.Contract` and can @@ -102,7 +107,7 @@ override three methods: .. method:: Contract.post_process(output) This allows processing the output of the callback. Iterators are - converted listified before being passed to this hook. + converted to lists before being passed to this hook. Raise :class:`~scrapy.exceptions.ContractFail` from :class:`~scrapy.contracts.Contract.pre_process` or @@ -111,22 +116,27 @@ Raise :class:`~scrapy.exceptions.ContractFail` from .. autoclass:: scrapy.exceptions.ContractFail Here is a demo contract which checks the presence of a custom header in the -response received:: +response received: + +.. skip: next +.. code-block:: python from scrapy.contracts import Contract from scrapy.exceptions import ContractFail + class HasHeaderContract(Contract): - """ Demo contract which checks the presence of a custom header - @has_header X-CustomHeader + """ + Demo contract which checks the presence of a custom header + @has_header X-CustomHeader """ - name = 'has_header' + name = "has_header" def pre_process(self, response): for header in self.args: if header not in response.headers: - raise ContractFail('X-CustomHeader not present') + raise ContractFail("X-CustomHeader not present") .. _detecting-contract-check-runs: @@ -135,14 +145,17 @@ Detecting check runs When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is set to the ``true`` string. You can use :data:`os.environ` to perform any change to -your spiders or your settings when ``scrapy check`` is used:: +your spiders or your settings when ``scrapy check`` is used: + +.. code-block:: python import os import scrapy + class ExampleSpider(scrapy.Spider): - name = 'example' + name = "example" def __init__(self): - if os.environ.get('SCRAPY_CHECK'): + if os.environ.get("SCRAPY_CHECK"): pass # Do some scraper adjustments when a check is running diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 2aef755c7..a65bab3ca 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -1,3 +1,5 @@ +.. _topics-coroutines: + ========== Coroutines ========== @@ -17,14 +19,12 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.Request` callbacks. - .. note:: The callback output is not processed until the whole callback - finishes. + If you are using any custom or third-party :ref:`spider middleware + `, see :ref:`sync-async-spider-middleware`. - As a side effect, if the callback raises an exception, none of its - output is processed. - - This is a known caveat of the current implementation that we aim to - address in a future version of Scrapy. + .. versionchanged:: 2.7 + Output of async callbacks is now processed asynchronously instead of + collecting all of it first. - The :meth:`process_item` method of :ref:`item pipelines `. @@ -39,54 +39,78 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. -Usage -===== +- The + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` + method of :ref:`spider middlewares `. -There are several use cases for coroutines in Scrapy. Code that would -return Deferreds when written for previous Scrapy versions, such as downloader -middlewares and signal handlers, can be rewritten to be shorter and cleaner:: + It must be defined as an :term:`asynchronous generator`. The input + ``result`` parameter is an :term:`asynchronous iterable`. + + See also :ref:`sync-async-spider-middleware` and + :ref:`universal-spider-middleware`. + + .. versionadded:: 2.7 + +General usage +============= + +There are several use cases for coroutines in Scrapy. + +Code that would return Deferreds when written for previous Scrapy versions, +such as downloader middlewares and signal handlers, can be rewritten to be +shorter and cleaner: + +.. code-block:: python from itemadapter import ItemAdapter + class DbPipeline: def _update_item(self, data, item): adapter = ItemAdapter(item) - adapter['field'] = data + adapter["field"] = data return item def process_item(self, item, spider): adapter = ItemAdapter(item) - dfd = db.get_some_data(adapter['id']) + dfd = db.get_some_data(adapter["id"]) dfd.addCallback(self._update_item, item) return dfd -becomes:: +becomes: + +.. code-block:: python from itemadapter import ItemAdapter + class DbPipeline: async def process_item(self, item, spider): adapter = ItemAdapter(item) - adapter['field'] = await db.get_some_data(adapter['id']) + adapter["field"] = await db.get_some_data(adapter["id"]) return item Coroutines may be used to call asynchronous code. This includes other coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. -This means you can use many useful Python libraries providing such code:: +This means you can use many useful Python libraries providing such code: + +.. skip: next +.. code-block:: python class MySpiderDeferred(Spider): # ... async def parse(self, response): - additional_response = await treq.get('https://additional.url') + additional_response = await treq.get("https://additional.url") additional_data = await treq.content(additional_response) # ... use response and additional_data to yield items and requests + class MySpiderAsyncio(Spider): # ... async def parse(self, response): async with aiohttp.ClientSession() as session: - async with session.get('https://additional.url') as additional_response: + async with session.get("https://additional.url") as additional_response: additional_data = await additional_response.text() # ... use response and additional_data to yield items and requests @@ -104,7 +128,159 @@ Common use cases for asynchronous code include: * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); -* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see - :ref:`the screenshot pipeline example`). +* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download` + (see :ref:`the screenshot pipeline example`). .. _aio-libs: https://github.com/aio-libs + + +.. _inline-requests: + +Inline requests +=============== + +The spider below shows how to send a request and await its response all from +within a spider callback: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + + + class SingleRequestSpider(Spider): + name = "single" + start_urls = ["https://example.org/product"] + + async def parse(self, response, **kwargs): + additional_request = Request("https://example.org/price") + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) + yield { + "h1": response.css("h1").get(), + "price": additional_response.css("#price").get(), + } + +You can also send multiple requests in parallel: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + from twisted.internet.defer import DeferredList + + + class MultipleRequestsSpider(Spider): + name = "multiple" + start_urls = ["https://example.com/product"] + + async def parse(self, response, **kwargs): + additional_requests = [ + Request("https://example.com/price"), + Request("https://example.com/color"), + ] + deferreds = [] + for r in additional_requests: + deferred = self.crawler.engine.download(r) + deferreds.append(deferred) + responses = await maybe_deferred_to_future(DeferredList(deferreds)) + yield { + "h1": response.css("h1::text").get(), + "price": responses[0][1].css(".price::text").get(), + "price2": responses[1][1].css(".color::text").get(), + } + + +.. _sync-async-spider-middleware: + +Mixing synchronous and asynchronous spider middlewares +====================================================== + +.. versionadded:: 2.7 + +The output of a :class:`~scrapy.Request` callback is passed as the ``result`` +parameter to the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method +of the first :ref:`spider middleware ` from the +:ref:`list of active spider middlewares `. +Then the output of that ``process_spider_output`` method is passed to the +``process_spider_output`` method of the next spider middleware, and so on for +every active spider middleware. + +Scrapy supports mixing :ref:`coroutine methods ` and synchronous methods +in this chain of calls. + +However, if any of the ``process_spider_output`` methods is defined as a +synchronous method, and the previous ``Request`` callback or +``process_spider_output`` method is a coroutine, there are some drawbacks to +the asynchronous-to-synchronous conversion that Scrapy does so that the +synchronous ``process_spider_output`` method gets a synchronous iterable as its +``result`` parameter: + +- The whole output of the previous ``Request`` callback or + ``process_spider_output`` method is awaited at this point. + +- If an exception raises while awaiting the output of the previous + ``Request`` callback or ``process_spider_output`` method, none of that + output will be processed. + + This contrasts with the regular behavior, where all items yielded before + an exception raises are processed. + +Asynchronous-to-synchronous conversions are supported for backward +compatibility, but they are deprecated and will stop working in a future +version of Scrapy. + +To avoid asynchronous-to-synchronous conversions, when defining ``Request`` +callbacks as coroutine methods or when using spider middlewares whose +``process_spider_output`` method is an :term:`asynchronous generator`, all +active spider middlewares must either have their ``process_spider_output`` +method defined as an asynchronous generator or :ref:`define a +process_spider_output_async method `. + +.. note:: When using third-party spider middlewares that only define a + synchronous ``process_spider_output`` method, consider + :ref:`making them universal ` through + :ref:`subclassing `. + + +.. _universal-spider-middleware: + +Universal spider middlewares +============================ + +.. versionadded:: 2.7 + +To allow writing a spider middleware that supports asynchronous execution of +its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding +:ref:`asynchronous-to-synchronous conversions `) +while maintaining support for older Scrapy versions, you may define +``process_spider_output`` as a synchronous method and define an +:term:`asynchronous generator` version of that method with an alternative name: +``process_spider_output_async``. + +For example: + +.. code-block:: python + + class UniversalSpiderMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: + # ... do something with r + yield r + + async def process_spider_output_async(self, response, result, spider): + async for r in result: + # ... do something with r + yield r + +.. note:: This is an interim measure to allow, for a time, to write code that + works in Scrapy 2.7 and later without requiring + asynchronous-to-synchronous conversions, and works in earlier Scrapy + versions as well. + + In some future version of Scrapy, however, this feature will be + deprecated and, eventually, in a later version of Scrapy, this + feature will be removed, and all spider middlewares will be expected + to define their ``process_spider_output`` method as an asynchronous + generator. diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 4d452b4df..49c5b0410 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -5,21 +5,25 @@ Debugging Spiders ================= This document explains the most common techniques for debugging spiders. -Consider the following Scrapy spider below:: +Consider the following Scrapy spider below: + +.. skip: next +.. code-block:: python import scrapy from myproject.items import MyItem + class MySpider(scrapy.Spider): - name = 'myspider' + name = "myspider" start_urls = ( - 'http://example.com/page1', - 'http://example.com/page2', - ) + "http://example.com/page1", + "http://example.com/page2", + ) def parse(self, response): # - # collect `item_urls` + # collect `item_urls` for item_url in item_urls: yield scrapy.Request(item_url, self.parse_item) @@ -28,7 +32,9 @@ Consider the following Scrapy spider below:: item = MyItem() # populate `item` fields # and extract item_details_url - yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item}) + yield scrapy.Request( + item_details_url, self.parse_details, cb_kwargs={"item": item} + ) def parse_details(self, response, item): # populate more `item` fields @@ -103,10 +109,13 @@ showing the response received and the output. How to debug the situation when .. highlight:: python Fortunately, the :command:`shell` is your bread and butter in this case (see -:ref:`topics-shell-inspect-response`):: +:ref:`topics-shell-inspect-response`): + +.. code-block:: python from scrapy.shell import inspect_response + def parse_details(self, response, item=None): if item: # populate more `item` fields @@ -121,10 +130,13 @@ Open in browser Sometimes you just want to see how a certain response looks in a browser, you can use the ``open_in_browser`` function for that. Here is an example of how -you would use it:: +you would use it: + +.. code-block:: python from scrapy.utils.response import open_in_browser + def parse_details(self, response): if "item name" not in response.body: open_in_browser(response) @@ -138,15 +150,47 @@ Logging Logging is another useful option for getting information about your spider run. Although not as convenient, it comes with the advantage that the logs will be -available in all future runs should they be necessary again:: +available in all future runs should they be necessary again: + +.. code-block:: python def parse_details(self, response, item=None): if item: # populate more `item` fields return item else: - self.logger.warning('No item received for %s', response.url) + self.logger.warning("No item received for %s", response.url) For more information, check the :ref:`topics-logging` section. .. _base tag: https://www.w3schools.com/tags/tag_base.asp + +.. _debug-vscode: + +Visual Studio Code +================== + +.. highlight:: json + +To debug spiders with Visual Studio Code you can use the following ``launch.json``:: + + { + "version": "0.1.0", + "configurations": [ + { + "name": "Python: Launch Scrapy Spider", + "type": "python", + "request": "launch", + "module": "scrapy", + "args": [ + "runspider", + "${file}" + ], + "console": "integratedTerminal" + } + ] + } + + +Also, make sure you enable "User Uncaught Exceptions", to catch exceptions in +your Scrapy spider. diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 9bf97c628..a15ee1059 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -94,8 +94,10 @@ Then, back to your web browser, right-click on the ``span`` tag, select response = load_response('https://quotes.toscrape.com/', 'quotes.html') ->>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() -['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] +.. code-block:: pycon + + >>> response.xpath("/html/body/div/div[2]/div[1]/div[1]/span[1]/text()").getall() + ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] Adding ``text()`` at the end we are able to extract the first quote with this basic selector. But this XPath is not really that clever. All it does is @@ -124,11 +126,13 @@ With this knowledge we can refine our XPath: Instead of a path to follow, we'll simply select all ``span`` tags with the ``class="text"`` by using the `has-class-extension`_: ->>> response.xpath('//span[has-class("text")]/text()').getall() -['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', -'“It is our choices, Harry, that show what we truly are, far more than our abilities.”', -'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', -...] +.. code-block:: pycon + + >>> response.xpath('//span[has-class("text")]/text()').getall() + ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', + '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', + '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', + ...] And with one simple, cleverer XPath we are able to extract all quotes from the page. We could have constructed a loop over our first XPath to increase @@ -237,17 +241,19 @@ on the request and open ``Open in new tab`` to get a better overview. :alt: JSON-object returned from the quotes.toscrape API With this response we can now easily parse the JSON-object and -also request each page to get every quote on the site:: +also request each page to get every quote on the site: + +.. code-block:: python import scrapy import json class QuoteSpider(scrapy.Spider): - name = 'quote' - allowed_domains = ['quotes.toscrape.com'] + name = "quote" + allowed_domains = ["quotes.toscrape.com"] page = 1 - start_urls = ['https://quotes.toscrape.com/api/quotes?page=1'] + start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"] def parse(self, response): data = json.loads(response.text) @@ -275,7 +281,9 @@ requests, as we could need to add ``headers`` or ``cookies`` to make it work. In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the :meth:`~scrapy.Request.from_curl()` method to generate an equivalent -request:: +request: + +.. code-block:: python from scrapy import Request @@ -286,7 +294,8 @@ request:: "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM" "zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW" "I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http" - "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") + "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'" + ) Alternatively, if you want to know the arguments needed to recreate that request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs` diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 29e350651..a8e5b23bf 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -17,10 +17,12 @@ To activate a downloader middleware component, add it to the :setting:`DOWNLOADER_MIDDLEWARES` setting, which is a dict whose keys are the middleware class paths and their values are the middleware orders. -Here's an example:: +Here's an example: + +.. code-block:: python DOWNLOADER_MIDDLEWARES = { - 'myproject.middlewares.CustomDownloaderMiddleware': 543, + "myproject.middlewares.CustomDownloaderMiddleware": 543, } The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the @@ -42,11 +44,13 @@ previous (or subsequent) middleware being applied. If you want to disable a built-in middleware (the ones defined in :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` -as its value. For example, if you want to disable the user-agent middleware:: +as its value. For example, if you want to disable the user-agent middleware: + +.. code-block:: python DOWNLOADER_MIDDLEWARES = { - 'myproject.middlewares.CustomDownloaderMiddleware': 543, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None, + "myproject.middlewares.CustomDownloaderMiddleware": 543, + "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -226,20 +230,26 @@ There is support for keeping multiple cookie sessions per spider by using the :reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar (session), but you can pass an identifier to use different ones. -For example:: +For example: + +.. skip: next +.. code-block:: python for i, url in enumerate(urls): - yield scrapy.Request(url, meta={'cookiejar': i}, - callback=self.parse_page) + yield scrapy.Request(url, meta={"cookiejar": i}, callback=self.parse_page) Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep -passing it along on subsequent requests. For example:: +passing it along on subsequent requests. For example: + +.. code-block:: python def parse_page(self, response): # do some processing - return scrapy.Request("http://www.example.com/otherpage", - meta={'cookiejar': response.meta['cookiejar']}, - callback=self.parse_other_page) + return scrapy.Request( + "http://www.example.com/otherpage", + meta={"cookiejar": response.meta["cookiejar"]}, + callback=self.parse_other_page, + ) .. setting:: COOKIES_ENABLED @@ -339,16 +349,18 @@ HttpAuthMiddleware domain of the first request, which will work for some spiders but not for others. In the future the middleware will produce an error instead. - Example:: + Example: + + .. code-block:: python from scrapy.spiders import CrawlSpider - class SomeIntranetSiteSpider(CrawlSpider): - http_user = 'someuser' - http_pass = 'somepass' - http_auth_domain = 'intranet.example.com' - name = 'intranet.example.com' + class SomeIntranetSiteSpider(CrawlSpider): + http_user = "someuser" + http_pass = "somepass" + http_auth_domain = "intranet.example.com" + name = "intranet.example.com" # .. rest of the spider code omitted ... @@ -792,7 +804,9 @@ If you want to handle some redirect status codes in your spider, you can specify these in the ``handle_httpstatus_list`` spider attribute. For example, if you want the redirect middleware to ignore 301 and 302 -responses (and pass them through to your spider) you can do this:: +responses (and pass them through to your spider) you can do this: + +.. code-block:: python class MySpider(CrawlSpider): handle_httpstatus_list = [301, 302] @@ -901,6 +915,7 @@ settings (see the settings documentation for more info): * :setting:`RETRY_ENABLED` * :setting:`RETRY_TIMES` * :setting:`RETRY_HTTP_CODES` +* :setting:`RETRY_EXCEPTIONS` .. reqmeta:: dont_retry @@ -952,10 +967,41 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_EXCEPTIONS + +RETRY_EXCEPTIONS +^^^^^^^^^^^^^^^^ + +Default:: + + [ + 'twisted.internet.defer.TimeoutError', + 'twisted.internet.error.TimeoutError', + 'twisted.internet.error.DNSLookupError', + 'twisted.internet.error.ConnectionRefusedError', + 'twisted.internet.error.ConnectionDone', + 'twisted.internet.error.ConnectError', + 'twisted.internet.error.ConnectionLost', + 'twisted.internet.error.TCPTimedOutError', + 'twisted.web.client.ResponseFailed', + IOError, + 'scrapy.core.downloader.handlers.http11.TunnelError', + ] + +List of exceptions to retry. + +Each list entry may be an exception type or its import path as a string. + +An exception will not be caught when the exception type is not in +:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request +has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught +exception propagation, see +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST ---------------------- +^^^^^^^^^^^^^^^^^^^^^ Default: ``-1`` @@ -1119,7 +1165,7 @@ In order to use this parser: .. _support-for-new-robots-parser: Implementing support for a new parser -------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can implement support for a new robots.txt_ parser by subclassing the abstract base class :class:`~scrapy.robotstxt.RobotParser` and diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index ea5d06210..a0f4b4411 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -119,16 +119,20 @@ data from it depends on the type of response: ` as usual. - If the response is JSON, use :func:`json.loads` to load the desired data from - :attr:`response.text `:: + :attr:`response.text `: + + .. code-block:: python data = json.loads(response.text) If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a :class:`~scrapy.Selector` and then - :ref:`use it ` as usual:: + :ref:`use it ` as usual: - selector = Selector(data['html']) + .. code-block:: python + + selector = Selector(data["html"]) - If the response is JavaScript, or HTML with a `` - """) - self.assertEqual(get_meta_refresh(r1), (5.0, 'http://example.org/newpage')) + """, + ) + self.assertEqual(get_meta_refresh(r1), (5.0, "http://example.org/newpage")) self.assertEqual(get_meta_refresh(r2), (None, None)) self.assertEqual(get_meta_refresh(r3), (None, None)) def test_get_base_url(self): - resp = HtmlResponse("http://www.example.com", body=b""" + resp = HtmlResponse( + "http://www.example.com", + body=b""" blahablsdfsal& - """) + """, + ) self.assertEqual(get_base_url(resp), "http://www.example.com/img/") - resp2 = HtmlResponse("http://www.example.com", body=b""" - blahablsdfsal&""") + resp2 = HtmlResponse( + "http://www.example.com", + body=b""" + blahablsdfsal&""", + ) self.assertEqual(get_base_url(resp2), "http://www.example.com") def test_response_status_message(self): - self.assertEqual(response_status_message(200), '200 OK') - self.assertEqual(response_status_message(404), '404 Not Found') + self.assertEqual(response_status_message(200), "200 OK") + self.assertEqual(response_status_message(404), "404 Not Found") self.assertEqual(response_status_message(573), "573 Unknown Status") def test_inject_base_url(self): @@ -89,38 +129,51 @@ class ResponseUtilsTest(unittest.TestCase): def check_base_url(burl): path = urlparse(burl).path - if not os.path.exists(path): - path = burl.replace('file://', '') - with open(path, "rb") as f: - bbody = f.read() + if not path or not Path(path).exists(): + path = burl.replace("file://", "") + bbody = Path(path).read_bytes() self.assertEqual(bbody.count(b''), 1) return True - r1 = HtmlResponse(url, body=b""" + r1 = HtmlResponse( + url, + body=b""" Dummy

Hello world.

- """) - r2 = HtmlResponse(url, body=b""" + """, + ) + r2 = HtmlResponse( + url, + body=b""" Dummy Hello world. - """) - r3 = HtmlResponse(url, body=b""" + """, + ) + r3 = HtmlResponse( + url, + body=b""" Dummy
Hello header

Hello world.

- """) - r4 = HtmlResponse(url, body=b""" + """, + ) + r4 = HtmlResponse( + url, + body=b""" Dummy

Hello world.

- """) - r5 = HtmlResponse(url, body=b""" + """, + ) + r5 = HtmlResponse( + url, + body=b"""

Hello world.

- """) + """, + ) assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" - assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head" - assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag" - assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment" - assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment" + assert open_in_browser( + r2, _openfunc=check_base_url + ), "Inject base url with argumented head" + assert open_in_browser( + r3, _openfunc=check_base_url + ), "Inject unique base url with misleading tag" + assert open_in_browser( + r4, _openfunc=check_base_url + ), "Inject unique base url with misleading comment" + assert open_in_browser( + r5, _openfunc=check_base_url + ), "Inject unique base url with conditional comment" diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index daf022aee..5cdcc7f7c 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,3 +1,4 @@ +import dataclasses import datetime import json import unittest @@ -10,14 +11,7 @@ from scrapy.http import Request, Response from scrapy.utils.serialize import ScrapyJSONEncoder -try: - from dataclasses import make_dataclass -except ImportError: - make_dataclass = None - - class JsonEncoderTestCase(unittest.TestCase): - def setUp(self): self.encoder = ScrapyJSONEncoder(sort_keys=True) @@ -30,19 +24,27 @@ class JsonEncoderTestCase(unittest.TestCase): ts = "10:11:12" dec = Decimal("1000.12") decs = "1000.12" - s = {'foo'} - ss = ['foo'] + s = {"foo"} + ss = ["foo"] dt_set = {dt} dt_sets = [dts] - for input, output in [('foo', 'foo'), (d, ds), (t, ts), (dt, dts), - (dec, decs), (['foo', d], ['foo', ds]), (s, ss), - (dt_set, dt_sets)]: - self.assertEqual(self.encoder.encode(input), - json.dumps(output, sort_keys=True)) + for input, output in [ + ("foo", "foo"), + (d, ds), + (t, ts), + (dt, dts), + (dec, decs), + (["foo", d], ["foo", ds]), + (s, ss), + (dt_set, dt_sets), + ]: + self.assertEqual( + self.encoder.encode(input), json.dumps(output, sort_keys=True) + ) def test_encode_deferred(self): - self.assertIn('Deferred', self.encoder.encode(defer.Deferred())) + self.assertIn("Deferred", self.encoder.encode(defer.Deferred())) def test_encode_request(self): r = Request("http://www.example.com/lala") @@ -56,17 +58,17 @@ class JsonEncoderTestCase(unittest.TestCase): self.assertIn(r.url, rs) self.assertIn(str(r.status), rs) - @unittest.skipIf(not make_dataclass, "No dataclass support") def test_encode_dataclass_item(self): - TestDataClass = make_dataclass( - "TestDataClass", - [("name", str), ("url", str), ("price", int)], - ) + @dataclasses.dataclass + class TestDataClass: + name: str + url: str + price: int + item = TestDataClass(name="Product", url="http://product.org", price=1) encoded = self.encoder.encode(item) self.assertEqual( - encoded, - '{"name": "Product", "price": 1, "url": "http://product.org"}' + encoded, '{"name": "Product", "price": 1, "url": "http://product.org"}' ) def test_encode_attrs_item(self): @@ -79,6 +81,5 @@ class JsonEncoderTestCase(unittest.TestCase): item = AttrsItem(name="Product", url="http://product.org", price=1) encoded = self.encoder.encode(item) self.assertEqual( - encoded, - '{"name": "Product", "price": 1, "url": "http://product.org"}' + encoded, '{"name": "Product", "price": 1, "url": "http://product.org"}' ) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index ad7394232..65b99e0c4 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,13 +1,10 @@ import asyncio -from unittest import SkipTest from pydispatch import dispatcher from pytest import mark from testfixtures import LogCapture -from twisted import version as twisted_version from twisted.internet import defer, reactor from twisted.python.failure import Failure -from twisted.python.versions import Version from twisted.trial import unittest from scrapy.utils.signal import send_catch_log, send_catch_log_deferred @@ -15,7 +12,6 @@ from scrapy.utils.test import get_from_asyncio_queue class SendCatchLogTest(unittest.TestCase): - @defer.inlineCallbacks def test_send_catch_log(self): test_signal = object() @@ -25,16 +21,18 @@ class SendCatchLogTest(unittest.TestCase): dispatcher.connect(self.ok_handler, signal=test_signal) with LogCapture() as log: result = yield defer.maybeDeferred( - self._get_result, test_signal, arg='test', - handlers_called=handlers_called + self._get_result, + test_signal, + arg="test", + handlers_called=handlers_called, ) assert self.error_handler in handlers_called assert self.ok_handler in handlers_called self.assertEqual(len(log.records), 1) record = log.records[0] - self.assertIn('error_handler', record.getMessage()) - self.assertEqual(record.levelname, 'ERROR') + self.assertIn("error_handler", record.getMessage()) + self.assertEqual(record.levelname, "ERROR") self.assertEqual(result[0][0], self.error_handler) self.assertIsInstance(result[0][1], Failure) self.assertEqual(result[1], (self.ok_handler, "OK")) @@ -51,71 +49,49 @@ class SendCatchLogTest(unittest.TestCase): def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) - assert arg == 'test' + assert arg == "test" return "OK" class SendCatchLogDeferredTest(SendCatchLogTest): - def _get_result(self, signal, *a, **kw): return send_catch_log_deferred(signal, *a, **kw) class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): - def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) - assert arg == 'test' + assert arg == "test" d = defer.Deferred() reactor.callLater(0, d.callback, "OK") return d -@mark.usefixtures('reactor_pytest') +@mark.usefixtures("reactor_pytest") class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): - async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) - assert arg == 'test' + assert arg == "test" await defer.succeed(42) return "OK" def test_send_catch_log(self): - if ( - self.reactor_pytest == 'asyncio' - and twisted_version < Version('twisted', 18, 4, 0) - ): - raise SkipTest( - 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' - 'fails due to a timeout when using AsyncIO and Twisted ' - 'versions lower than 18.4.0' - ) - return super().test_send_catch_log() @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): - async def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) - assert arg == 'test' + assert arg == "test" await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") def test_send_catch_log(self): - if twisted_version < Version('twisted', 18, 4, 0): - raise SkipTest( - 'Due to https://twistedmatrix.com/trac/ticket/9390, this test ' - 'fails due to a timeout when using Twisted versions lower ' - 'than 18.4.0' - ) - return super().test_send_catch_log() class SendCatchLogTest2(unittest.TestCase): - def test_error_logged_if_deferred_not_supported(self): def test_handler(): return defer.Deferred() diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 23eb261b7..ce0de0722 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -4,9 +4,9 @@ from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots class SitemapTest(unittest.TestCase): - def test_sitemap(self): - s = Sitemap(b""" + s = Sitemap( + b""" http://www.example.com/ @@ -20,19 +20,30 @@ class SitemapTest(unittest.TestCase): weekly 0.8 -""") - assert s.type == 'urlset' +""" + ) + assert s.type == "urlset" self.assertEqual( list(s), [ - {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', - 'lastmod': '2009-08-16', 'changefreq': 'weekly'}, - ] + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + { + "priority": "0.8", + "loc": "http://www.example.com/Special-Offers.html", + "lastmod": "2009-08-16", + "changefreq": "weekly", + }, + ], ) def test_sitemap_index(self): - s = Sitemap(b""" + s = Sitemap( + b""" http://www.example.com/sitemap1.xml.gz @@ -42,21 +53,29 @@ class SitemapTest(unittest.TestCase): http://www.example.com/sitemap2.xml.gz 2005-01-01 -""") - assert s.type == 'sitemapindex' +""" + ) + assert s.type == "sitemapindex" self.assertEqual( list(s), [ - {'loc': 'http://www.example.com/sitemap1.xml.gz', 'lastmod': '2004-10-01T18:23:17+00:00'}, - {'loc': 'http://www.example.com/sitemap2.xml.gz', 'lastmod': '2005-01-01'}, - ] + { + "loc": "http://www.example.com/sitemap1.xml.gz", + "lastmod": "2004-10-01T18:23:17+00:00", + }, + { + "loc": "http://www.example.com/sitemap2.xml.gz", + "lastmod": "2005-01-01", + }, + ], ) def test_sitemap_strip(self): """Assert we can deal with trailing spaces inside tags - we've seen those """ - s = Sitemap(b""" + s = Sitemap( + b""" http://www.example.com/ @@ -69,19 +88,26 @@ class SitemapTest(unittest.TestCase): -""") +""" + ) self.assertEqual( list(s), [ - {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'loc': 'http://www.example.com/2', 'lastmod': ''}, - ] + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + {"loc": "http://www.example.com/2", "lastmod": ""}, + ], ) def test_sitemap_wrong_ns(self): """We have seen sitemaps with wrongs ns. Presumably, Google still works with these, though is not 100% confirmed""" - s = Sitemap(b""" + s = Sitemap( + b""" http://www.example.com/ @@ -94,19 +120,26 @@ class SitemapTest(unittest.TestCase): -""") +""" + ) self.assertEqual( list(s), [ - {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'loc': 'http://www.example.com/2', 'lastmod': ''}, - ] + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + {"loc": "http://www.example.com/2", "lastmod": ""}, + ], ) def test_sitemap_wrong_ns2(self): """We have seen sitemaps with wrongs ns. Presumably, Google still works with these, though is not 100% confirmed""" - s = Sitemap(b""" + s = Sitemap( + b""" http://www.example.com/ @@ -119,14 +152,20 @@ class SitemapTest(unittest.TestCase): -""") - assert s.type == 'urlset' +""" + ) + assert s.type == "urlset" self.assertEqual( list(s), [ - {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, - {'loc': 'http://www.example.com/2', 'lastmod': ''}, - ] + { + "priority": "1", + "loc": "http://www.example.com/", + "lastmod": "2009-08-16", + "changefreq": "daily", + }, + {"loc": "http://www.example.com/2", "lastmod": ""}, + ], ) def test_sitemap_urls_from_robots(self): @@ -148,15 +187,20 @@ Sitemap: /sitemap-relative-url.xml Disallow: /forum/search/ Disallow: /forum/active/ """ - self.assertEqual(list(sitemap_urls_from_robots(robots, base_url='http://example.com')), - ['http://example.com/sitemap.xml', - 'http://example.com/sitemap-product-index.xml', - 'http://example.com/sitemap-uppercase.xml', - 'http://example.com/sitemap-relative-url.xml']) + self.assertEqual( + list(sitemap_urls_from_robots(robots, base_url="http://example.com")), + [ + "http://example.com/sitemap.xml", + "http://example.com/sitemap-product-index.xml", + "http://example.com/sitemap-uppercase.xml", + "http://example.com/sitemap-relative-url.xml", + ], + ) def test_sitemap_blanklines(self): """Assert we can deal with starting blank lines before tag""" - s = Sitemap(b""" + s = Sitemap( + b""" @@ -178,29 +222,34 @@ Disallow: /forum/active/ -""") - self.assertEqual(list(s), [ - {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap1.xml'}, - {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap2.xml'}, - {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap3.xml'}, - ]) +""" + ) + self.assertEqual( + list(s), + [ + {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap1.xml"}, + {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap2.xml"}, + {"lastmod": "2013-07-15", "loc": "http://www.example.com/sitemap3.xml"}, + ], + ) def test_comment(self): - s = Sitemap(b""" + s = Sitemap( + b""" http://www.example.com/ - """) + """ + ) - self.assertEqual(list(s), [ - {'loc': 'http://www.example.com/'} - ]) + self.assertEqual(list(s), [{"loc": "http://www.example.com/"}]) def test_alternate(self): - s = Sitemap(b""" + s = Sitemap( + b""" @@ -213,24 +262,26 @@ Disallow: /forum/active/ href="http://www.example.com/english/"/> - """) + """ + ) self.assertEqual( list(s), [ { - 'loc': 'http://www.example.com/english/', - 'alternate': [ - 'http://www.example.com/deutsch/', - 'http://www.example.com/schweiz-deutsch/', - 'http://www.example.com/english/', + "loc": "http://www.example.com/english/", + "alternate": [ + "http://www.example.com/deutsch/", + "http://www.example.com/schweiz-deutsch/", + "http://www.example.com/english/", ], } - ] + ], ) def test_xml_entity_expansion(self): - s = Sitemap(b""" + s = Sitemap( + b""" @@ -240,10 +291,11 @@ Disallow: /forum/active/ http://127.0.0.1:8000/&xxe; - """) + """ + ) - self.assertEqual(list(s), [{'loc': 'http://127.0.0.1:8000/'}]) + self.assertEqual(list(s), [{"loc": "http://127.0.0.1:8000/"}]) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 3c87268ab..460ae40c3 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -3,22 +3,21 @@ import unittest from scrapy import Spider from scrapy.http import Request from scrapy.item import Item -from scrapy.utils.spider import iterate_spider_output, iter_spider_classes +from scrapy.utils.spider import iter_spider_classes, iterate_spider_output class MySpider1(Spider): - name = 'myspider1' + name = "myspider1" class MySpider2(Spider): - name = 'myspider2' + name = "myspider2" class UtilsSpidersTestCase(unittest.TestCase): - def test_iterate_spider_output(self): i = Item() - r = Request('http://scrapytest.org') + r = Request("http://scrapytest.org") o = object() self.assertEqual(list(iterate_spider_output(i)), [i]) @@ -28,6 +27,7 @@ class UtilsSpidersTestCase(unittest.TestCase): def test_iter_spider_classes(self): import tests.test_utils_spider + it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 1d5e63363..cbe80e157 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,15 +1,14 @@ -import os +import unittest +from pathlib import Path from shutil import rmtree from tempfile import mkdtemp -import unittest + from scrapy.utils.template import render_templatefile - -__doctests__ = ['scrapy.utils.template'] +__doctests__ = ["scrapy.utils.template"] class UtilsRenderTemplateFileTestCase(unittest.TestCase): - def setUp(self): self.tmp_path = mkdtemp() @@ -17,27 +16,24 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): rmtree(self.tmp_path) def test_simple_render(self): + context = dict(project_name="proj", name="spi", classname="TheSpider") + template = "from ${project_name}.spiders.${name} import ${classname}" + rendered = "from proj.spiders.spi import TheSpider" - context = dict(project_name='proj', name='spi', classname='TheSpider') - template = 'from ${project_name}.spiders.${name} import ${classname}' - rendered = 'from proj.spiders.spi import TheSpider' + template_path = Path(self.tmp_path, "templ.py.tmpl") + render_path = Path(self.tmp_path, "templ.py") - template_path = os.path.join(self.tmp_path, 'templ.py.tmpl') - render_path = os.path.join(self.tmp_path, 'templ.py') - - with open(template_path, 'wb') as tmpl_file: - tmpl_file.write(template.encode('utf8')) - assert os.path.isfile(template_path) # Failure of test itself + template_path.write_text(template, encoding="utf8") + assert template_path.is_file() # Failure of test itself render_templatefile(template_path, **context) - self.assertFalse(os.path.exists(template_path)) - with open(render_path, 'rb') as result: - self.assertEqual(result.read().decode('utf8'), rendered) + self.assertFalse(template_path.exists()) + self.assertEqual(render_path.read_text(encoding="utf8"), rendered) - os.remove(render_path) - assert not os.path.exists(render_path) # Failure of test itself + render_path.unlink() + assert not render_path.exists() # Failure of test itself -if '__main__' == __name__: +if "__main__" == __name__: unittest.main() diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index b8e8c3130..35d1508c6 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -17,7 +17,6 @@ class Bar(trackref.object_ref): class TrackrefTestCase(unittest.TestCase): - def setUp(self): trackref.live_refs.clear() @@ -27,34 +26,39 @@ class TrackrefTestCase(unittest.TestCase): o3 = Foo() # NOQA self.assertEqual( trackref.format_live_refs(), - '''\ + """\ Live References Bar 1 oldest: 0s ago Foo 2 oldest: 0s ago -''') +""", + ) self.assertEqual( trackref.format_live_refs(ignore=Foo), - '''\ + """\ Live References Bar 1 oldest: 0s ago -''') +""", + ) - @mock.patch('sys.stdout', new_callable=StringIO) + @mock.patch("sys.stdout", new_callable=StringIO) def test_print_live_refs_empty(self, stdout): trackref.print_live_refs() - self.assertEqual(stdout.getvalue(), 'Live References\n\n\n') + self.assertEqual(stdout.getvalue(), "Live References\n\n\n") - @mock.patch('sys.stdout', new_callable=StringIO) + @mock.patch("sys.stdout", new_callable=StringIO) def test_print_live_refs_with_objects(self, stdout): o1 = Foo() # NOQA trackref.print_live_refs() - self.assertEqual(stdout.getvalue(), '''\ + self.assertEqual( + stdout.getvalue(), + """\ Live References -Foo 1 oldest: 0s ago\n\n''') +Foo 1 oldest: 0s ago\n\n""", + ) def test_get_oldest(self): o1 = Foo() # NOQA @@ -68,18 +72,18 @@ Foo 1 oldest: 0s ago\n\n''') sleep(0.01) o3_time = time() if o3_time <= o1_time: - raise SkipTest('time.time is not precise enough') + raise SkipTest("time.time is not precise enough") o3 = Foo() # NOQA - self.assertIs(trackref.get_oldest('Foo'), o1) - self.assertIs(trackref.get_oldest('Bar'), o2) - self.assertIsNone(trackref.get_oldest('XXX')) + self.assertIs(trackref.get_oldest("Foo"), o1) + self.assertIs(trackref.get_oldest("Bar"), o2) + self.assertIsNone(trackref.get_oldest("XXX")) def test_iter_all(self): o1 = Foo() # NOQA o2 = Bar() # NOQA o3 = Foo() # NOQA self.assertEqual( - set(trackref.iter_all('Foo')), + set(trackref.iter_all("Foo")), {o1, o3}, ) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 144c7bd76..65522f0fd 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,208 +1,313 @@ import unittest +from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider +from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import ( + _is_filesystem_path, add_http_if_no_scheme, guess_scheme, - _is_filesystem_path, strip_url, + url_has_any_extension, url_is_from_any_domain, url_is_from_spider, ) - -__doctests__ = ['scrapy.utils.url'] +__doctests__ = ["scrapy.utils.url"] class UrlUtilsTest(unittest.TestCase): - def test_url_is_from_any_domain(self): - url = 'http://www.wheele-bin-art.co.uk/get/product/123' - self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) - self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) + url = "http://www.wheele-bin-art.co.uk/get/product/123" + self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])) + self.assertFalse(url_is_from_any_domain(url, ["art.co.uk"])) - url = 'http://wheele-bin-art.co.uk/get/product/123' - self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) - self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) + url = "http://wheele-bin-art.co.uk/get/product/123" + self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.co.uk"])) + self.assertFalse(url_is_from_any_domain(url, ["art.co.uk"])) - url = 'http://www.Wheele-Bin-Art.co.uk/get/product/123' - self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.CO.UK'])) - self.assertTrue(url_is_from_any_domain(url, ['WHEELE-BIN-ART.CO.UK'])) + url = "http://www.Wheele-Bin-Art.co.uk/get/product/123" + self.assertTrue(url_is_from_any_domain(url, ["wheele-bin-art.CO.UK"])) + self.assertTrue(url_is_from_any_domain(url, ["WHEELE-BIN-ART.CO.UK"])) - url = 'http://192.169.0.15:8080/mypage.html' - self.assertTrue(url_is_from_any_domain(url, ['192.169.0.15:8080'])) - self.assertFalse(url_is_from_any_domain(url, ['192.169.0.15'])) + url = "http://192.169.0.15:8080/mypage.html" + self.assertTrue(url_is_from_any_domain(url, ["192.169.0.15:8080"])) + self.assertFalse(url_is_from_any_domain(url, ["192.169.0.15"])) url = ( - 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20' - 'javascript:%20document.orderform_2581_1190810811.submit%28%29' + "javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20" + "javascript:%20document.orderform_2581_1190810811.submit%28%29" + ) + self.assertFalse(url_is_from_any_domain(url, ["testdomain.com"])) + self.assertFalse( + url_is_from_any_domain(url + ".testdomain.com", ["testdomain.com"]) ) - self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) - self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com'])) def test_url_is_from_spider(self): - spider = Spider(name='example.com') - self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) - self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider)) - self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', spider)) - self.assertFalse(url_is_from_spider('http://www.example.net/some/page.html', spider)) + spider = Spider(name="example.com") + self.assertTrue( + url_is_from_spider("http://www.example.com/some/page.html", spider) + ) + self.assertTrue( + url_is_from_spider("http://sub.example.com/some/page.html", spider) + ) + self.assertFalse( + url_is_from_spider("http://www.example.org/some/page.html", spider) + ) + self.assertFalse( + url_is_from_spider("http://www.example.net/some/page.html", spider) + ) def test_url_is_from_spider_class_attributes(self): class MySpider(Spider): - name = 'example.com' - self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider)) - self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider)) - self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', MySpider)) - self.assertFalse(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) + name = "example.com" + + self.assertTrue( + url_is_from_spider("http://www.example.com/some/page.html", MySpider) + ) + self.assertTrue( + url_is_from_spider("http://sub.example.com/some/page.html", MySpider) + ) + self.assertFalse( + url_is_from_spider("http://www.example.org/some/page.html", MySpider) + ) + self.assertFalse( + url_is_from_spider("http://www.example.net/some/page.html", MySpider) + ) def test_url_is_from_spider_with_allowed_domains(self): - spider = Spider(name='example.com', allowed_domains=['example.org', 'example.net']) - self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) - self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider)) - self.assertTrue(url_is_from_spider('http://example.com/some/page.html', spider)) - self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', spider)) - self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider)) - self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider)) + spider = Spider( + name="example.com", allowed_domains=["example.org", "example.net"] + ) + self.assertTrue( + url_is_from_spider("http://www.example.com/some/page.html", spider) + ) + self.assertTrue( + url_is_from_spider("http://sub.example.com/some/page.html", spider) + ) + self.assertTrue(url_is_from_spider("http://example.com/some/page.html", spider)) + self.assertTrue( + url_is_from_spider("http://www.example.org/some/page.html", spider) + ) + self.assertTrue( + url_is_from_spider("http://www.example.net/some/page.html", spider) + ) + self.assertFalse( + url_is_from_spider("http://www.example.us/some/page.html", spider) + ) - spider = Spider(name='example.com', allowed_domains={'example.com', 'example.net'}) - self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) + spider = Spider( + name="example.com", allowed_domains={"example.com", "example.net"} + ) + self.assertTrue( + url_is_from_spider("http://www.example.com/some/page.html", spider) + ) - spider = Spider(name='example.com', allowed_domains=('example.com', 'example.net')) - self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) + spider = Spider( + name="example.com", allowed_domains=("example.com", "example.net") + ) + self.assertTrue( + url_is_from_spider("http://www.example.com/some/page.html", spider) + ) def test_url_is_from_spider_with_allowed_domains_class_attributes(self): class MySpider(Spider): - name = 'example.com' - allowed_domains = ('example.org', 'example.net') - self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider)) - self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider)) - self.assertTrue(url_is_from_spider('http://example.com/some/page.html', MySpider)) - self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', MySpider)) - self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) - self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider)) + name = "example.com" + allowed_domains = ("example.org", "example.net") + + self.assertTrue( + url_is_from_spider("http://www.example.com/some/page.html", MySpider) + ) + self.assertTrue( + url_is_from_spider("http://sub.example.com/some/page.html", MySpider) + ) + self.assertTrue( + url_is_from_spider("http://example.com/some/page.html", MySpider) + ) + self.assertTrue( + url_is_from_spider("http://www.example.org/some/page.html", MySpider) + ) + self.assertTrue( + url_is_from_spider("http://www.example.net/some/page.html", MySpider) + ) + self.assertFalse( + url_is_from_spider("http://www.example.us/some/page.html", MySpider) + ) + + def test_url_has_any_extension(self): + deny_extensions = {"." + e for e in arg_to_iter(IGNORED_EXTENSIONS)} + self.assertTrue( + url_has_any_extension( + "http://www.example.com/archive.tar.gz", deny_extensions + ) + ) + self.assertTrue( + url_has_any_extension("http://www.example.com/page.doc", deny_extensions) + ) + self.assertTrue( + url_has_any_extension("http://www.example.com/page.pdf", deny_extensions) + ) + self.assertFalse( + url_has_any_extension("http://www.example.com/page.htm", deny_extensions) + ) + self.assertFalse( + url_has_any_extension("http://www.example.com/", deny_extensions) + ) + self.assertFalse( + url_has_any_extension( + "http://www.example.com/page.doc.html", deny_extensions + ) + ) class AddHttpIfNoScheme(unittest.TestCase): - def test_add_scheme(self): - self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com') + self.assertEqual( + add_http_if_no_scheme("www.example.com"), "http://www.example.com" + ) def test_without_subdomain(self): - self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com') + self.assertEqual(add_http_if_no_scheme("example.com"), "http://example.com") def test_path(self): self.assertEqual( - add_http_if_no_scheme('www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + add_http_if_no_scheme("www.example.com/some/page.html"), + "http://www.example.com/some/page.html", + ) def test_port(self): self.assertEqual( - add_http_if_no_scheme('www.example.com:80'), - 'http://www.example.com:80') + add_http_if_no_scheme("www.example.com:80"), "http://www.example.com:80" + ) def test_fragment(self): self.assertEqual( - add_http_if_no_scheme('www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + add_http_if_no_scheme("www.example.com/some/page#frag"), + "http://www.example.com/some/page#frag", + ) def test_query(self): self.assertEqual( - add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + add_http_if_no_scheme("www.example.com/do?a=1&b=2&c=3"), + "http://www.example.com/do?a=1&b=2&c=3", + ) def test_username_password(self): self.assertEqual( - add_http_if_no_scheme('username:password@www.example.com'), - 'http://username:password@www.example.com') + add_http_if_no_scheme("username:password@www.example.com"), + "http://username:password@www.example.com", + ) def test_complete_url(self): self.assertEqual( - add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + add_http_if_no_scheme( + "username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" + ), + "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag", + ) def test_preserve_http(self): - self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com') + self.assertEqual( + add_http_if_no_scheme("http://www.example.com"), "http://www.example.com" + ) def test_preserve_http_without_subdomain(self): self.assertEqual( - add_http_if_no_scheme('http://example.com'), - 'http://example.com') + add_http_if_no_scheme("http://example.com"), "http://example.com" + ) def test_preserve_http_path(self): self.assertEqual( - add_http_if_no_scheme('http://www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + add_http_if_no_scheme("http://www.example.com/some/page.html"), + "http://www.example.com/some/page.html", + ) def test_preserve_http_port(self): self.assertEqual( - add_http_if_no_scheme('http://www.example.com:80'), - 'http://www.example.com:80') + add_http_if_no_scheme("http://www.example.com:80"), + "http://www.example.com:80", + ) def test_preserve_http_fragment(self): self.assertEqual( - add_http_if_no_scheme('http://www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + add_http_if_no_scheme("http://www.example.com/some/page#frag"), + "http://www.example.com/some/page#frag", + ) def test_preserve_http_query(self): self.assertEqual( - add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + add_http_if_no_scheme("http://www.example.com/do?a=1&b=2&c=3"), + "http://www.example.com/do?a=1&b=2&c=3", + ) def test_preserve_http_username_password(self): self.assertEqual( - add_http_if_no_scheme('http://username:password@www.example.com'), - 'http://username:password@www.example.com') + add_http_if_no_scheme("http://username:password@www.example.com"), + "http://username:password@www.example.com", + ) def test_preserve_http_complete_url(self): self.assertEqual( - add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + add_http_if_no_scheme( + "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" + ), + "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag", + ) def test_protocol_relative(self): self.assertEqual( - add_http_if_no_scheme('//www.example.com'), 'http://www.example.com') + add_http_if_no_scheme("//www.example.com"), "http://www.example.com" + ) def test_protocol_relative_without_subdomain(self): - self.assertEqual( - add_http_if_no_scheme('//example.com'), 'http://example.com') + self.assertEqual(add_http_if_no_scheme("//example.com"), "http://example.com") def test_protocol_relative_path(self): self.assertEqual( - add_http_if_no_scheme('//www.example.com/some/page.html'), - 'http://www.example.com/some/page.html') + add_http_if_no_scheme("//www.example.com/some/page.html"), + "http://www.example.com/some/page.html", + ) def test_protocol_relative_port(self): self.assertEqual( - add_http_if_no_scheme('//www.example.com:80'), - 'http://www.example.com:80') + add_http_if_no_scheme("//www.example.com:80"), "http://www.example.com:80" + ) def test_protocol_relative_fragment(self): self.assertEqual( - add_http_if_no_scheme('//www.example.com/some/page#frag'), - 'http://www.example.com/some/page#frag') + add_http_if_no_scheme("//www.example.com/some/page#frag"), + "http://www.example.com/some/page#frag", + ) def test_protocol_relative_query(self): self.assertEqual( - add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'), - 'http://www.example.com/do?a=1&b=2&c=3') + add_http_if_no_scheme("//www.example.com/do?a=1&b=2&c=3"), + "http://www.example.com/do?a=1&b=2&c=3", + ) def test_protocol_relative_username_password(self): self.assertEqual( - add_http_if_no_scheme('//username:password@www.example.com'), - 'http://username:password@www.example.com') + add_http_if_no_scheme("//username:password@www.example.com"), + "http://username:password@www.example.com", + ) def test_protocol_relative_complete_url(self): self.assertEqual( - add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), - 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + add_http_if_no_scheme( + "//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag" + ), + "http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag", + ) def test_preserve_https(self): self.assertEqual( - add_http_if_no_scheme('https://www.example.com'), - 'https://www.example.com') + add_http_if_no_scheme("https://www.example.com"), "https://www.example.com" + ) def test_preserve_ftp(self): - self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com') + self.assertEqual( + add_http_if_no_scheme("ftp://www.example.com"), "ftp://www.example.com" + ) class GuessSchemeTest(unittest.TestCase): @@ -212,8 +317,10 @@ class GuessSchemeTest(unittest.TestCase): def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) - assert url.startswith(args[1]), \ - f'Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`' + assert url.startswith( + args[1] + ), f"Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`" + return do_expected @@ -222,100 +329,100 @@ def create_skipped_scheme_t(args): raise unittest.SkipTest(args[2]) url = guess_scheme(args[0]) assert url.startswith(args[1]) + return do_expected for k, args in enumerate( [ - ('/index', 'file://'), - ('/index.html', 'file://'), - ('./index.html', 'file://'), - ('../index.html', 'file://'), - ('../../index.html', 'file://'), - ('./data/index.html', 'file://'), - ('.hidden/data/index.html', 'file://'), - ('/home/user/www/index.html', 'file://'), - ('//home/user/www/index.html', 'file://'), - ('file:///home/user/www/index.html', 'file://'), - - ('index.html', 'http://'), - ('example.com', 'http://'), - ('www.example.com', 'http://'), - ('www.example.com/index.html', 'http://'), - ('http://example.com', 'http://'), - ('http://example.com/index.html', 'http://'), - ('localhost', 'http://'), - ('localhost/index.html', 'http://'), - + ("/index", "file://"), + ("/index.html", "file://"), + ("./index.html", "file://"), + ("../index.html", "file://"), + ("../../index.html", "file://"), + ("./data/index.html", "file://"), + (".hidden/data/index.html", "file://"), + ("/home/user/www/index.html", "file://"), + ("//home/user/www/index.html", "file://"), + ("file:///home/user/www/index.html", "file://"), + ("index.html", "http://"), + ("example.com", "http://"), + ("www.example.com", "http://"), + ("www.example.com/index.html", "http://"), + ("http://example.com", "http://"), + ("http://example.com/index.html", "http://"), + ("localhost", "http://"), + ("localhost/index.html", "http://"), # some corner cases (default to http://) - ('/', 'http://'), - ('.../test', 'http://'), + ("/", "http://"), + (".../test", "http://"), ], start=1, ): t_method = create_guess_scheme_t(args) - t_method.__name__ = f'test_uri_{k:03}' + t_method.__name__ = f"test_uri_{k:03}" setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation for k, args in enumerate( [ ( - r'C:\absolute\path\to\a\file.html', - 'file://', - 'Windows filepath are not supported for scrapy shell', + r"C:\absolute\path\to\a\file.html", + "file://", + "Windows filepath are not supported for scrapy shell", ), ], start=1, ): t_method = create_skipped_scheme_t(args) - t_method.__name__ = f'test_uri_skipped_{k:03}' + t_method.__name__ = f"test_uri_skipped_{k:03}" setattr(GuessSchemeTest, t_method.__name__, t_method) class StripUrl(unittest.TestCase): - def test_noop(self): - self.assertEqual(strip_url( - 'http://www.example.com/index.html'), - 'http://www.example.com/index.html') + self.assertEqual( + strip_url("http://www.example.com/index.html"), + "http://www.example.com/index.html", + ) def test_noop_query_string(self): - self.assertEqual(strip_url( - 'http://www.example.com/index.html?somekey=somevalue'), - 'http://www.example.com/index.html?somekey=somevalue') + self.assertEqual( + strip_url("http://www.example.com/index.html?somekey=somevalue"), + "http://www.example.com/index.html?somekey=somevalue", + ) def test_fragments(self): - self.assertEqual(strip_url( - 'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False), - 'http://www.example.com/index.html?somekey=somevalue#section') + self.assertEqual( + strip_url( + "http://www.example.com/index.html?somekey=somevalue#section", + strip_fragment=False, + ), + "http://www.example.com/index.html?somekey=somevalue#section", + ) def test_path(self): for input_url, origin, output_url in [ - ('http://www.example.com/', - False, - 'http://www.example.com/'), - - ('http://www.example.com', - False, - 'http://www.example.com'), - - ('http://www.example.com', - True, - 'http://www.example.com/'), + ("http://www.example.com/", False, "http://www.example.com/"), + ("http://www.example.com", False, "http://www.example.com"), + ("http://www.example.com", True, "http://www.example.com/"), ]: self.assertEqual(strip_url(input_url, origin_only=origin), output_url) def test_credentials(self): for i, o in [ - ('http://username@www.example.com/index.html?somekey=somevalue#section', - 'http://www.example.com/index.html?somekey=somevalue'), - - ('https://username:@www.example.com/index.html?somekey=somevalue#section', - 'https://www.example.com/index.html?somekey=somevalue'), - - ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section', - 'ftp://www.example.com/index.html?somekey=somevalue'), + ( + "http://username@www.example.com/index.html?somekey=somevalue#section", + "http://www.example.com/index.html?somekey=somevalue", + ), + ( + "https://username:@www.example.com/index.html?somekey=somevalue#section", + "https://www.example.com/index.html?somekey=somevalue", + ), + ( + "ftp://username:password@www.example.com/index.html?somekey=somevalue#section", + "ftp://www.example.com/index.html?somekey=somevalue", + ), ]: self.assertEqual(strip_url(i, strip_credentials=True), o) @@ -323,124 +430,163 @@ class StripUrl(unittest.TestCase): for i, o in [ # user: "username@" # password: none - ('http://username%40@www.example.com/index.html?somekey=somevalue#section', - 'http://www.example.com/index.html?somekey=somevalue'), - + ( + "http://username%40@www.example.com/index.html?somekey=somevalue#section", + "http://www.example.com/index.html?somekey=somevalue", + ), # user: "username:pass" # password: "" - ('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section', - 'https://www.example.com/index.html?somekey=somevalue'), - + ( + "https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section", + "https://www.example.com/index.html?somekey=somevalue", + ), # user: "me" # password: "user@domain.com" - ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section', - 'ftp://www.example.com/index.html?somekey=somevalue'), + ( + "ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section", + "ftp://www.example.com/index.html?somekey=somevalue", + ), ]: self.assertEqual(strip_url(i, strip_credentials=True), o) def test_default_ports_creds_off(self): for i, o in [ - ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section', - 'http://www.example.com/index.html?somekey=somevalue'), - - ('http://username:password@www.example.com:8080/index.html#section', - 'http://www.example.com:8080/index.html'), - - ('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section', - 'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'), - - ('https://username:password@www.example.com:443/index.html', - 'https://www.example.com/index.html'), - - ('https://username:password@www.example.com:442/index.html', - 'https://www.example.com:442/index.html'), - - ('https://username:password@www.example.com:80/index.html', - 'https://www.example.com:80/index.html'), - - ('ftp://username:password@www.example.com:21/file.txt', - 'ftp://www.example.com/file.txt'), - - ('ftp://username:password@www.example.com:221/file.txt', - 'ftp://www.example.com:221/file.txt'), + ( + "http://username:password@www.example.com:80/index.html?somekey=somevalue#section", + "http://www.example.com/index.html?somekey=somevalue", + ), + ( + "http://username:password@www.example.com:8080/index.html#section", + "http://www.example.com:8080/index.html", + ), + ( + "http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section", + "http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov", + ), + ( + "https://username:password@www.example.com:443/index.html", + "https://www.example.com/index.html", + ), + ( + "https://username:password@www.example.com:442/index.html", + "https://www.example.com:442/index.html", + ), + ( + "https://username:password@www.example.com:80/index.html", + "https://www.example.com:80/index.html", + ), + ( + "ftp://username:password@www.example.com:21/file.txt", + "ftp://www.example.com/file.txt", + ), + ( + "ftp://username:password@www.example.com:221/file.txt", + "ftp://www.example.com:221/file.txt", + ), ]: self.assertEqual(strip_url(i), o) def test_default_ports(self): for i, o in [ - ('http://username:password@www.example.com:80/index.html', - 'http://username:password@www.example.com/index.html'), - - ('http://username:password@www.example.com:8080/index.html', - 'http://username:password@www.example.com:8080/index.html'), - - ('http://username:password@www.example.com:443/index.html', - 'http://username:password@www.example.com:443/index.html'), - - ('https://username:password@www.example.com:443/index.html', - 'https://username:password@www.example.com/index.html'), - - ('https://username:password@www.example.com:442/index.html', - 'https://username:password@www.example.com:442/index.html'), - - ('https://username:password@www.example.com:80/index.html', - 'https://username:password@www.example.com:80/index.html'), - - ('ftp://username:password@www.example.com:21/file.txt', - 'ftp://username:password@www.example.com/file.txt'), - - ('ftp://username:password@www.example.com:221/file.txt', - 'ftp://username:password@www.example.com:221/file.txt'), + ( + "http://username:password@www.example.com:80/index.html", + "http://username:password@www.example.com/index.html", + ), + ( + "http://username:password@www.example.com:8080/index.html", + "http://username:password@www.example.com:8080/index.html", + ), + ( + "http://username:password@www.example.com:443/index.html", + "http://username:password@www.example.com:443/index.html", + ), + ( + "https://username:password@www.example.com:443/index.html", + "https://username:password@www.example.com/index.html", + ), + ( + "https://username:password@www.example.com:442/index.html", + "https://username:password@www.example.com:442/index.html", + ), + ( + "https://username:password@www.example.com:80/index.html", + "https://username:password@www.example.com:80/index.html", + ), + ( + "ftp://username:password@www.example.com:21/file.txt", + "ftp://username:password@www.example.com/file.txt", + ), + ( + "ftp://username:password@www.example.com:221/file.txt", + "ftp://username:password@www.example.com:221/file.txt", + ), ]: - self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) + self.assertEqual( + strip_url(i, strip_default_port=True, strip_credentials=False), o + ) def test_default_ports_keep(self): for i, o in [ - ('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section', - 'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'), - - ('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section', - 'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'), - - ('http://username:password@www.example.com:443/index.html', - 'http://username:password@www.example.com:443/index.html'), - - ('https://username:password@www.example.com:443/index.html', - 'https://username:password@www.example.com:443/index.html'), - - ('https://username:password@www.example.com:442/index.html', - 'https://username:password@www.example.com:442/index.html'), - - ('https://username:password@www.example.com:80/index.html', - 'https://username:password@www.example.com:80/index.html'), - - ('ftp://username:password@www.example.com:21/file.txt', - 'ftp://username:password@www.example.com:21/file.txt'), - - ('ftp://username:password@www.example.com:221/file.txt', - 'ftp://username:password@www.example.com:221/file.txt'), + ( + "http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section", + "http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov", + ), + ( + "http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section", + "http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov", + ), + ( + "http://username:password@www.example.com:443/index.html", + "http://username:password@www.example.com:443/index.html", + ), + ( + "https://username:password@www.example.com:443/index.html", + "https://username:password@www.example.com:443/index.html", + ), + ( + "https://username:password@www.example.com:442/index.html", + "https://username:password@www.example.com:442/index.html", + ), + ( + "https://username:password@www.example.com:80/index.html", + "https://username:password@www.example.com:80/index.html", + ), + ( + "ftp://username:password@www.example.com:21/file.txt", + "ftp://username:password@www.example.com:21/file.txt", + ), + ( + "ftp://username:password@www.example.com:221/file.txt", + "ftp://username:password@www.example.com:221/file.txt", + ), ]: - self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) + self.assertEqual( + strip_url(i, strip_default_port=False, strip_credentials=False), o + ) def test_origin_only(self): for i, o in [ - ('http://username:password@www.example.com/index.html', - 'http://www.example.com/'), - - ('http://username:password@www.example.com:80/foo/bar?query=value#somefrag', - 'http://www.example.com/'), - - ('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag', - 'http://www.example.com:8008/'), - - ('https://username:password@www.example.com:443/index.html', - 'https://www.example.com/'), + ( + "http://username:password@www.example.com/index.html", + "http://www.example.com/", + ), + ( + "http://username:password@www.example.com:80/foo/bar?query=value#somefrag", + "http://www.example.com/", + ), + ( + "http://username:password@www.example.com:8008/foo/bar?query=value#somefrag", + "http://www.example.com:8008/", + ), + ( + "https://username:password@www.example.com:443/index.html", + "https://www.example.com/", + ), ]: self.assertEqual(strip_url(i, origin_only=True), o) class IsPathTestCase(unittest.TestCase): - def test_path(self): for input_value, output_value in ( # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell @@ -456,10 +602,11 @@ class IsPathTestCase(unittest.TestCase): (r"\\?\UNC\Server01\user\docs\Letter.txt", True), (r"\\?\C:\user\docs\Letter.txt", True), (r"C:\user\docs\somefile.ext:alternate_stream_name", True), - (r"https://example.com", False), ): - self.assertEqual(_is_filesystem_path(input_value), output_value, input_value) + self.assertEqual( + _is_filesystem_path(input_value), output_value, input_value + ) if __name__ == "__main__": diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a6d55cb38..0042fe8f0 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -2,29 +2,27 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ -import os import shutil -import sys -from pkg_resources import parse_version +from pathlib import Path -import cryptography import OpenSSL.SSL +from twisted.internet import defer, reactor from twisted.trial import unittest -from twisted.web import server, static, util, resource -from twisted.internet import reactor, defer +from twisted.web import resource, server, static, util + try: from twisted.internet.testing import StringTransport except ImportError: # deprecated in Twisted 19.7.0 # (remove once we bump our requirement past that version) from twisted.test.proto_helpers import StringTransport -from twisted.python.filepath import FilePath -from twisted.protocols.policies import WrappingFactory + from twisted.internet.defer import inlineCallbacks +from twisted.protocols.policies import WrappingFactory from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory -from scrapy.http import Request, Headers +from scrapy.http import Headers, Request from scrapy.settings import Settings from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes, to_unicode @@ -41,17 +39,24 @@ from tests.mockserver import ( def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): """Adapted version of twisted.web.client.getPage""" + def _clientfactory(url, *args, **kwargs): url = to_unicode(url) - timeout = kwargs.pop('timeout', 0) + timeout = kwargs.pop("timeout", 0) f = client.ScrapyHTTPClientFactory( - Request(url, *args, **kwargs), timeout=timeout) + Request(url, *args, **kwargs), timeout=timeout + ) f.deferred.addCallback(response_transform or (lambda r: r.body)) return f from twisted.web.client import _makeGetterFactory + return _makeGetterFactory( - to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs + to_bytes(url), + _clientfactory, + contextFactory=contextFactory, + *args, + **kwargs, ).deferred @@ -63,51 +68,80 @@ class ParseUrlTestCase(unittest.TestCase): return (f.scheme, f.netloc, f.host, f.port, f.path) def testParse(self): - lip = '127.0.0.1' + lip = "127.0.0.1" tests = ( - ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), - ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), - ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')), - ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')), - - ("http://127.0.0.1", ('http', lip, lip, 80, '/')), - ("http://127.0.0.1/", ('http', lip, lip, 80, '/')), - ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')), - ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')), - ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')), - ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')), - ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')), - ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')), - - ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')), - ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')), - ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')), - - ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')), - ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')), + ( + "http://127.0.0.1?c=v&c2=v2#fragment", + ("http", lip, lip, 80, "/?c=v&c2=v2"), + ), + ( + "http://127.0.0.1/?c=v&c2=v2#fragment", + ("http", lip, lip, 80, "/?c=v&c2=v2"), + ), + ( + "http://127.0.0.1/foo?c=v&c2=v2#frag", + ("http", lip, lip, 80, "/foo?c=v&c2=v2"), + ), + ( + "http://127.0.0.1:100?c=v&c2=v2#fragment", + ("http", lip + ":100", lip, 100, "/?c=v&c2=v2"), + ), + ( + "http://127.0.0.1:100/?c=v&c2=v2#frag", + ("http", lip + ":100", lip, 100, "/?c=v&c2=v2"), + ), + ( + "http://127.0.0.1:100/foo?c=v&c2=v2#frag", + ("http", lip + ":100", lip, 100, "/foo?c=v&c2=v2"), + ), + ("http://127.0.0.1", ("http", lip, lip, 80, "/")), + ("http://127.0.0.1/", ("http", lip, lip, 80, "/")), + ("http://127.0.0.1/foo", ("http", lip, lip, 80, "/foo")), + ("http://127.0.0.1?param=value", ("http", lip, lip, 80, "/?param=value")), + ("http://127.0.0.1/?param=value", ("http", lip, lip, 80, "/?param=value")), + ( + "http://127.0.0.1:12345/foo", + ("http", lip + ":12345", lip, 12345, "/foo"), + ), + ("http://spam:12345/foo", ("http", "spam:12345", "spam", 12345, "/foo")), + ( + "http://spam.test.org/foo", + ("http", "spam.test.org", "spam.test.org", 80, "/foo"), + ), + ("https://127.0.0.1/foo", ("https", lip, lip, 443, "/foo")), + ( + "https://127.0.0.1/?param=value", + ("https", lip, lip, 443, "/?param=value"), + ), + ("https://127.0.0.1:12345/", ("https", lip + ":12345", lip, 12345, "/")), + ( + "http://scrapytest.org/foo ", + ("http", "scrapytest.org", "scrapytest.org", 80, "/foo"), + ), + ("http://egg:7890 ", ("http", "egg:7890", "egg", 7890, "/")), ) for url, test in tests: - test = tuple( - to_bytes(x) if not isinstance(x, int) else x for x in test) + test = tuple(to_bytes(x) if not isinstance(x, int) else x for x in test) self.assertEqual(client._parse(url), test, url) class ScrapyHTTPPageGetterTests(unittest.TestCase): - def test_earlyHeaders(self): # basic test stolen from twisted HTTPageGetter - factory = client.ScrapyHTTPClientFactory(Request( - url='http://foo/bar', - body="some data", - headers={ - 'Host': 'example.net', - 'User-Agent': 'fooble', - 'Cookie': 'blah blah', - 'Content-Length': '12981', - 'Useful': 'value'})) + factory = client.ScrapyHTTPClientFactory( + Request( + url="http://foo/bar", + body="some data", + headers={ + "Host": "example.net", + "User-Agent": "fooble", + "Cookie": "blah blah", + "Content-Length": "12981", + "Useful": "value", + }, + ) + ) self._test( factory, @@ -119,22 +153,22 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): b"Host: example.net\r\n" b"Cookie: blah blah\r\n" b"\r\n" - b"some data") + b"some data", + ) # test minimal sent headers - factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar')) - self._test( - factory, - b"GET /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"\r\n") + factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar")) + self._test(factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n") # test a simple POST with body and content-type - factory = client.ScrapyHTTPClientFactory(Request( - method='POST', - url='http://foo/bar', - body='name=value', - headers={'Content-Type': 'application/x-www-form-urlencoded'})) + factory = client.ScrapyHTTPClientFactory( + Request( + method="POST", + url="http://foo/bar", + body="name=value", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + ) self._test( factory, @@ -144,29 +178,29 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): b"Content-Type: application/x-www-form-urlencoded\r\n" b"Content-Length: 10\r\n" b"\r\n" - b"name=value") + b"name=value", + ) # test a POST method with no body provided - factory = client.ScrapyHTTPClientFactory(Request( - method='POST', - url='http://foo/bar' - )) + factory = client.ScrapyHTTPClientFactory( + Request(method="POST", url="http://foo/bar") + ) self._test( factory, - b"POST /bar HTTP/1.0\r\n" - b"Host: foo\r\n" - b"Content-Length: 0\r\n" - b"\r\n") + b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" b"\r\n", + ) # test with single and multivalued headers - factory = client.ScrapyHTTPClientFactory(Request( - url='http://foo/bar', - headers={ - 'X-Meta-Single': 'single', - 'X-Meta-Multivalued': ['value1', 'value2'], - }, - )) + factory = client.ScrapyHTTPClientFactory( + Request( + url="http://foo/bar", + headers={ + "X-Meta-Single": "single", + "X-Meta-Multivalued": ["value1", "value2"], + }, + ) + ) self._test( factory, @@ -175,16 +209,21 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): b"X-Meta-Multivalued: value1\r\n" b"X-Meta-Multivalued: value2\r\n" b"X-Meta-Single: single\r\n" - b"\r\n") + b"\r\n", + ) # same test with single and multivalued headers but using Headers class - factory = client.ScrapyHTTPClientFactory(Request( - url='http://foo/bar', - headers=Headers({ - 'X-Meta-Single': 'single', - 'X-Meta-Multivalued': ['value1', 'value2'], - }), - )) + factory = client.ScrapyHTTPClientFactory( + Request( + url="http://foo/bar", + headers=Headers( + { + "X-Meta-Single": "single", + "X-Meta-Multivalued": ["value1", "value2"], + } + ), + ) + ) self._test( factory, @@ -193,7 +232,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): b"X-Meta-Multivalued: value1\r\n" b"X-Meta-Multivalued: value2\r\n" b"X-Meta-Single: single\r\n" - b"\r\n") + b"\r\n", + ) def _test(self, factory, testvalue): transport = StringTransport() @@ -201,14 +241,13 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol.factory = factory protocol.makeConnection(transport) self.assertEqual( - set(transport.value().splitlines()), - set(testvalue.splitlines())) + set(transport.value().splitlines()), set(testvalue.splitlines()) + ) return testvalue def test_non_standard_line_endings(self): # regression test for: http://dev.scrapy.org/ticket/258 - factory = client.ScrapyHTTPClientFactory(Request( - url='http://foo/bar')) + factory = client.ScrapyHTTPClientFactory(Request(url="http://foo/bar")) protocol = client.ScrapyHTTPPageGetter() protocol.factory = factory protocol.headers = Headers() @@ -216,15 +255,17 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol.dataReceived(b"Hello: World\n") protocol.dataReceived(b"Foo: Bar\n") protocol.dataReceived(b"\n") - self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']})) + self.assertEqual( + protocol.headers, Headers({"Hello": ["World"], "Foo": ["Bar"]}) + ) class EncodingResource(resource.Resource): - out_encoding = 'cp1251' + out_encoding = "cp1251" def render(self, request): body = to_unicode(request.content.read()) - request.setHeader(b'content-encoding', self.out_encoding) + request.setHeader(b"content-encoding", self.out_encoding) return body.encode(self.out_encoding) @@ -233,10 +274,10 @@ class WebClientTestCase(unittest.TestCase): return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): - self.tmpname = self.mktemp() - os.mkdir(self.tmpname) - FilePath(self.tmpname).child("file").setContent(b"0123456789") - r = static.File(self.tmpname) + self.tmpname = Path(self.mktemp()) + self.tmpname.mkdir() + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"error", ErrorResource()) @@ -261,16 +302,22 @@ class WebClientTestCase(unittest.TestCase): def testPayload(self): s = "0123456789" * 10 return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEqual, to_bytes(s)) + self.assertEqual, to_bytes(s) + ) def testHostHeader(self): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url - return defer.gatherResults([ - getPage(self.getURL("host")).addCallback( - self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")), - getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( - self.assertEqual, to_bytes("www.example.com"))]) + return defer.gatherResults( + [ + getPage(self.getURL("host")).addCallback( + self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}") + ), + getPage( + self.getURL("host"), headers={"Host": "www.example.com"} + ).addCallback(self.assertEqual, to_bytes("www.example.com")), + ] + ) def test_getPage(self): """ @@ -287,11 +334,16 @@ class WebClientTestCase(unittest.TestCase): the empty string if the method is C{HEAD} and there is a successful response code. """ + def _getPage(method): return getPage(self.getURL("file"), method=method) - return defer.gatherResults([ - _getPage("head").addCallback(self.assertEqual, b""), - _getPage("HEAD").addCallback(self.assertEqual, b"")]) + + return defer.gatherResults( + [ + _getPage("head").addCallback(self.assertEqual, b""), + _getPage("HEAD").addCallback(self.assertEqual, b""), + ] + ) def test_timeoutNotTriggering(self): """ @@ -300,8 +352,7 @@ class WebClientTestCase(unittest.TestCase): called back with the contents of the page. """ d = getPage(self.getURL("host"), timeout=100) - d.addCallback( - self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")) + d.addCallback(self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")) return d def test_timeoutTriggering(self): @@ -311,8 +362,8 @@ class WebClientTestCase(unittest.TestCase): L{Deferred} is errbacked with a L{error.TimeoutError}. """ finished = self.assertFailure( - getPage(self.getURL("wait"), timeout=0.000001), - defer.TimeoutError) + getPage(self.getURL("wait"), timeout=0.000001), defer.TimeoutError + ) def cleanup(passthrough): # Clean up the server which is hanging around not doing @@ -323,27 +374,28 @@ class WebClientTestCase(unittest.TestCase): if connected: connected[0].transport.loseConnection() return passthrough + finished.addBoth(cleanup) return finished def testNotFound(self): - return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) + return getPage(self.getURL("notsuchfile")).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): - self.assertIn(b'404 - No Such Resource', pageData) + self.assertIn(b"404 - No Such Resource", pageData) def testFactoryInfo(self): - url = self.getURL('file') + url = self.getURL("file") _, _, host, port, _ = client._parse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) reactor.connectTCP(to_unicode(host), port, factory) return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): - self.assertEqual(factory.status, b'200') - self.assertTrue(factory.version.startswith(b'HTTP/')) - self.assertEqual(factory.message, b'OK') - self.assertEqual(factory.response_headers[b'content-length'], b'10') + self.assertEqual(factory.status, b"200") + self.assertTrue(factory.version.startswith(b"HTTP/")) + self.assertEqual(factory.message, b"OK") + self.assertEqual(factory.response_headers[b"content-length"], b"10") def testRedirect(self): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) @@ -353,20 +405,24 @@ class WebClientTestCase(unittest.TestCase): pageData, b'\n\n \n \n' b' \n \n ' - b'click here\n \n\n') + b'click here\n \n\n', + ) def test_encoding(self): - """ Test that non-standart body encoding matches - Content-Encoding header """ - body = b'\xd0\x81\xd1\x8e\xd0\xaf' - dfd = getPage(self.getURL('encoding'), body=body, response_transform=lambda r: r) + """Test that non-standart body encoding matches + Content-Encoding header""" + body = b"\xd0\x81\xd1\x8e\xd0\xaf" + dfd = getPage( + self.getURL("encoding"), body=body, response_transform=lambda r: r + ) return dfd.addCallback(self._check_Encoding, body) def _check_Encoding(self, response, original_body): - content_encoding = to_unicode(response.headers[b'Content-Encoding']) + content_encoding = to_unicode(response.headers[b"Content-Encoding"]) self.assertEqual(content_encoding, EncodingResource.out_encoding) self.assertEqual( - response.body.decode(content_encoding), to_unicode(original_body)) + response.body.decode(content_encoding), to_unicode(original_body) + ) class WebClientSSLTestCase(unittest.TestCase): @@ -374,18 +430,20 @@ class WebClientSSLTestCase(unittest.TestCase): def _listen(self, site): return reactor.listenSSL( - 0, site, + 0, + site, contextFactory=self.context_factory or ssl_context_factory(), - interface="127.0.0.1") + interface="127.0.0.1", + ) def getURL(self, path): return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): - self.tmpname = self.mktemp() - os.mkdir(self.tmpname) - FilePath(self.tmpname).child("file").setContent(b"0123456789") - r = static.File(self.tmpname) + self.tmpname = Path(self.mktemp()) + self.tmpname.mkdir() + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) r.putChild(b"payload", PayloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) @@ -400,27 +458,34 @@ class WebClientSSLTestCase(unittest.TestCase): def testPayload(self): s = "0123456789" * 10 return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEqual, to_bytes(s)) + self.assertEqual, to_bytes(s) + ) class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): # we try to use a cipher that is not enabled by default in OpenSSL - custom_ciphers = 'CAMELLIA256-SHA' + custom_ciphers = "CAMELLIA256-SHA" context_factory = ssl_context_factory(cipher_string=custom_ciphers) def testPayload(self): s = "0123456789" * 10 - settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers}) - client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) + settings = Settings({"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers}) + client_context_factory = create_instance( + ScrapyClientContextFactory, settings=settings, crawler=None + ) return getPage( self.getURL("payload"), body=s, contextFactory=client_context_factory ).addCallback(self.assertEqual, to_bytes(s)) def testPayloadDisabledCipher(self): - if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"): - self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1") s = "0123456789" * 10 - settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) - client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) - d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory) + settings = Settings( + {"DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"} + ) + client_context_factory = create_instance( + ScrapyClientContextFactory, settings=settings, crawler=None + ) + d = getPage( + self.getURL("payload"), body=s, contextFactory=client_context_factory + ) return self.assertFailure(d, OpenSSL.SSL.Error) diff --git a/tox.ini b/tox.ini index 6951b6d16..ec3a59366 100644 --- a/tox.ini +++ b/tox.ini @@ -4,24 +4,18 @@ # and then run "tox" from this directory. [tox] -envlist = security,flake8,py +envlist = pre-commit,pylint,typing,py minversion = 1.7.0 [testenv] deps = -rtests/requirements.txt # mitmproxy does not support PyPy - # mitmproxy does not support Windows when running Python < 3.7 # Python 3.9+ requires mitmproxy >= 5.3.0 # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 - mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' - mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' - # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) - markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' - # Extras - botocore>=1.4.87 + mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -32,108 +26,118 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules install_command = - pip install -U -ctests/upper-constraints.txt {opts} {packages} + python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} [testenv:typing] basepython = python3 deps = - lxml-stubs==0.2.0 - mypy==0.910 - types-pyOpenSSL==20.0.3 - types-setuptools==57.0.0 + mypy==1.2.0 + types-attrs==19.1.0 + types-lxml==2023.3.28 + types-Pillow==9.4.0.19 + types-Pygments==2.14.0.7 + types-pyOpenSSL==23.1.0.1 + types-setuptools==67.6.0.7 commands = - pip install types-dataclasses # remove once py36 support is dropped mypy --show-error-codes {posargs: scrapy tests} -[testenv:security] +[testenv:pre-commit] basepython = python3 deps = - bandit==1.7.3 + pre-commit commands = - bandit -r -c .bandit.yml {posargs:scrapy} - -[testenv:flake8] -basepython = python3 -deps = - {[testenv]deps} - # Twisted[http2] is required to import some files - Twisted[http2]>=17.9.0 - pytest-flake8 - flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 -commands = - pytest --flake8 {posargs:docs scrapy tests} + pre-commit run {posargs:--all-files} [testenv:pylint] basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.12.2 + pylint==2.17.2 commands = pylint conftest.py docs extras scrapy setup.py tests +[testenv:twinecheck] +basepython = python3 +deps = + twine==4.0.2 + build==0.10.0 +commands = + python -m build --sdist + twine check dist/* + [pinned] deps = - cryptography==2.0 + cryptography==36.0.0 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==16.2.0 + pyOpenSSL==21.0.0 queuelib==1.4.2 - service_identity==16.0.0 - Twisted[http2]==17.9.0 + service_identity==18.1.0 + Twisted[http2]==18.9.0 w3lib==1.17.0 - zope.interface==4.1.3 + zope.interface==5.1.0 + lxml==4.4.1 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment - - # Extras - botocore==1.4.87 - google-cloud-storage==1.29.0 - Pillow==4.0.0 setenv = _SCRAPY_PINNED=true install_command = - pip install -U {opts} {packages} + python -I -m pip install {opts} {packages} +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] +basepython = python3.8 deps = {[pinned]deps} - lxml==3.5.0 PyDispatcher==2.0.5 install_command = {[pinned]install_command} setenv = {[pinned]setenv} +commands = {[pinned]commands} [testenv:windows-pinned] basepython = python3 deps = {[pinned]deps} - # First lxml version that includes a Windows wheel for Python 3.6, so we do - # not need to build lxml from sources in a CI Windows job: - lxml==3.8.0 PyDispatcher==2.0.5 install_command = {[pinned]install_command} setenv = {[pinned]setenv} +commands = {[pinned]commands} [testenv:extra-deps] +basepython = python3 deps = {[testenv]deps} - boto + boto3 google-cloud-storage # Twisted[http2] currently forces old mitmproxy because of h2 version # restrictions in their deps, so we need to pin old markupsafe here too. markupsafe < 2.1.0 - reppy robotexclusionrulesparser - Pillow>=4.0.0 - Twisted[http2]>=17.9.0 + Pillow + Twisted[http2] + +[testenv:extra-deps-pinned] +basepython = python3.8 +deps = + {[pinned]deps} + boto3==1.20.0 + google-cloud-storage==1.29.0 + Pillow==7.1.0 + robotexclusionrulesparser==1.6.2 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} +commands = {[pinned]commands} [testenv:asyncio] commands = @@ -141,7 +145,7 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} -commands = {[testenv:asyncio]commands} +commands = {[pinned]commands} --reactor=asyncio install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -155,9 +159,9 @@ commands = basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} - lxml==4.0.0 PyPyDispatcher==2.1.0 -commands = {[testenv:pypy3]commands} +commands = + pytest --durations=10 scrapy tests install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -193,3 +197,24 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck + + +# Run S3 tests with botocore installed but without boto3. + +[testenv:botocore] +deps = + {[testenv]deps} + botocore>=1.4.87 +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + +[testenv:botocore-pinned] +basepython = python3.8 +deps = + {[pinned]deps} + botocore==1.4.87 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}