diff --git a/.bandit.yml b/.bandit.yml deleted file mode 100644 index 2aae8a0aa..000000000 --- a/.bandit.yml +++ /dev/null @@ -1,21 +0,0 @@ -skips: -- B101 -- B113 # https://github.com/PyCQA/bandit/issues/1010 -- B105 -- B301 -- B303 -- B306 -- B307 -- B311 -- B320 -- B321 -- B324 -- B402 # https://github.com/scrapy/scrapy/issues/4180 -- B403 -- B404 -- B406 -- B410 -- B503 -- B603 -- B605 -exclude_dirs: ['tests'] diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 0bcd1ab01..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[bumpversion] -current_version = 2.10.1 -commit = True -tag = True -tag_name = {new_version} - -[bumpversion:file:scrapy/VERSION] diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index ad0ee0f6c..000000000 --- a/.coveragerc +++ /dev/null @@ -1,6 +0,0 @@ -[run] -branch = true -include = scrapy/* -omit = - tests/* -disable_warnings = include-ignored diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 544d72956..000000000 --- a/.flake8 +++ /dev/null @@ -1,22 +0,0 @@ -[flake8] - -max-line-length = 119 -ignore = W503, E203 - -exclude = - docs/conf.py - -per-file-ignores = -# Exclude files that are meant to provide top-level imports -# E402: Module level import not at top of file -# F401: Module imported but unused - scrapy/__init__.py:E402 - scrapy/core/downloader/handlers/http.py:F401 - scrapy/http/__init__.py:F401 - scrapy/linkextractors/__init__.py:E402,F401 - scrapy/selector/__init__.py:F401 - scrapy/spiders/__init__.py:E402,F401 - - # Issues pending a review: - scrapy/utils/url.py:F403,F405 - tests/test_loader.py:E741 diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index dbcebfa0a..1f062eef2 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,7 +1,7 @@ # .git-blame-ignore-revs # adding black formatter to all the code e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d -# re applying black to the code with default line length +# reapplying black to the code with default line length 303f0a70fcf8067adf0a909c2096a5009162383a -# reaplying black again and removing line length on pre-commit black config -c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 \ No newline at end of file +# reapplying black again and removing line length on pre-commit black config +c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..63cae77e7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,13 @@ +--- +name: Question / Help +about: Ask a question about Scrapy or ask for help with your Scrapy code. +--- + +Thanks for taking an interest in Scrapy! + +The Scrapy GitHub issue tracker is not meant for questions or help. Please ask +for help in the [Scrapy community resources](https://scrapy.org/community/) +instead. + +The GitHub issue tracker's purpose is to deal with bug reports and feature +requests for the project itself. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..623d48cfa --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: monthly + groups: + github-actions: + patterns: + - "*" + cooldown: + default-days: 7 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ee0cb4b1e..331fade61 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,42 +1,76 @@ name: Checks -on: [push, pull_request] + +permissions: + contents: read + +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: + +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true jobs: checks: runs-on: ubuntu-latest + env: + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: include: - - python-version: "3.11" + - python-version: "3.14" env: TOXENV: pylint - - python-version: 3.8 + - python-version: "3.10" env: - TOXENV: typing - - python-version: "3.11" # Keep in sync with .readthedocs.yml + TOXENV: mypy + - python-version: "3.10" + env: + TOXENV: mypy-tests + # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version. + - python-version: "3.14" env: TOXENV: docs - - python-version: "3.11" + - python-version: "3.13" + env: + TOXENV: docs-tests + - python-version: "3.14" env: TOXENV: twinecheck steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + docs/requirements.txt + pyproject.toml + tox.ini + - name: Run check env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: pre-commit/action@v3.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..c50576ec7 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,59 @@ +--- +name: codspeed + +on: + push: + branches: + - master + pull_request: + paths: + - scrapy/** + - tests/benchmarks/** + - .github/workflows/codspeed.yml + - tox.ini + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: {} + +jobs: + benchmark: + runs-on: ubuntu-latest + env: + # Make uv use the interpreter that actions/setup-python installed + # instead of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system + permissions: + contents: read + id-token: write # OIDC authentication with CodSpeed + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install dependencies + # tox must stay on PATH for the CodSpeed action to invoke it. + run: | + uv tool install --with tox-uv tox + tox -n -e benchmark + - name: Run benchmarks + uses: CodSpeedHQ/action@0ca9cbbf4623b599a6c3ed4fc8a922942705d9f1 # v5.0.2 + with: + mode: simulation + run: tox -e benchmark diff --git a/.github/workflows/flag-prs-for-triage.yml b/.github/workflows/flag-prs-for-triage.yml new file mode 100644 index 000000000..5ef8fa24a --- /dev/null +++ b/.github/workflows/flag-prs-for-triage.yml @@ -0,0 +1,255 @@ +name: Flag PRs for triage +# Labels pull requests whose author's public activity suggests that an LLM is +# writing them without supervision, and records the evidence in the workflow +# run summary so that triaging one does not require reading a user profile. +# +# Four independent signals, any of which is enough to label. Each one abstains +# when the data it needs is unavailable, so a missing signal never counts +# against an author: +# +# - Rejection burst: pull requests of theirs closed unmerged elsewhere within +# the last month. Volume of rejections in absolute terms separates spraying +# from ordinary contribution far better than a merge ratio does, since +# ratios reward authors who accumulate merges in trivial repositories. +# - Spray breadth: unrelated repositories they open pull requests against +# within one week. Breadth catches an agent on its first day, before any of +# its pull requests have been closed, and it comes from the event feed, so it +# also covers authors that the search API refuses to return. +# - Assistant voice: their recent comments across GitHub read as assistant +# output rather than as a developer talking, by section headings, bullet +# lists, em dash density or stock acknowledgement phrases. +# - Agent branch: the branch name carries an agent prefix. +# +# Authors that the organisations behind this repository already trust are left +# alone before any of that runs: public members of those organisations, and +# authors with a track record of pull requests merged into their repositories. +# Trust from a merge record rather than from a list of names keeps the exemption +# in step with who is actually contributing. +# +# Deliberately not used: account age, fork age, follower count, total pull +# request count and cross-repository merge ratio. All of them were measured +# against hand-labelled pull requests and either failed to separate or, in the +# case of the merge ratio, inverted on held-out data. +# +# The label is advisory, and it says the author's history is worth a look +# before reviewing in depth; it does not say the pull request is bad. +# +# The workflow only reads pull request and public activity metadata through the +# API, it never checks out or runs pull request code, so pull_request_target is +# safe here. +on: # zizmor: ignore[dangerous-triggers] + pull_request_target: + types: [opened] +permissions: + contents: read + pull-requests: write +jobs: + flag-pr-for-triage: + name: Label PR if the author's activity suggests unsupervised LLM use + runs-on: ubuntu-latest + steps: + - name: Score the author and label the PR + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const LABEL = 'needs triage'; + const RETRIES = 5; + const RETRY_WAIT_MS = 60000; + const REJECTION_WINDOW_DAYS = 30; + const MIN_REJECTIONS = 1; + const MIN_COMMENTS = 2; + const MAX_REPOS_PER_WEEK = 2; + const VOICE = { structure: 0.10, emDashPerKChar: 0.30, acknowledgement: 0.40 }; + const EVENT_PAGES = 3; + const TRUSTED_ORGS = ['scrapy', 'scrapy-plugins', 'scrapinghub', 'zytedata']; + const MIN_TRUSTED_MERGES = 10; + const AGENT_BRANCH = /^(agent|codex|claude|cursor|devin|copilot|jules|bot)[\/_-]/i; + + const { owner, repo } = context.repo; + const pr = context.payload.pull_request; + const author = pr.user.login; + + if (pr.user.type === 'Bot' + || ['MEMBER', 'OWNER', 'COLLABORATOR'].includes(pr.author_association)) { + core.info(`Skipping PR #${pr.number} by ${author} (${pr.user.type}, ${pr.author_association}).`); + return; + } + + // Rate and abuse limits reset on the order of a minute, so waiting + // is enough; other errors are not worth retrying. + const retriable = new Set([403, 429, 500, 502, 503, 504]); + const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + async function withRetries(description, call) { + for (let attempt = 1; ; attempt++) { + try { + return await call(); + } catch (error) { + if (!retriable.has(error.status) || attempt > RETRIES) throw error; + const reset = Number(error.response?.headers?.['x-ratelimit-reset']) * 1000 - Date.now(); + const after = Number(error.response?.headers?.['retry-after']) * 1000; + const wait = Math.min(Math.max(after || reset || RETRY_WAIT_MS, RETRY_WAIT_MS), 15 * RETRY_WAIT_MS); + core.info(`${description} failed with ${error.status}, retrying in ${Math.round(wait / 1000)}s (attempt ${attempt}/${RETRIES}).`); + await sleep(wait); + } + } + } + // Accounts excluded from search, deleted users and the like leave a + // signal unmeasurable rather than negative. + const orNull = promise => promise.catch(error => { + if ([404, 410, 422].includes(error.status)) return null; + throw error; + }); + + // author_association only reports membership of the organisation + // that owns this repository, and only when it is public, so trust + // in the author is established here instead. + const trustedOrg = (await Promise.all(TRUSTED_ORGS.map(org => + orNull(withRetries(`Checking public membership of ${org}`, () => + github.rest.orgs.checkPublicMembershipForUser({ org, username: author }), + )).then(response => response && org), + ))).find(Boolean); + if (trustedOrg) { + core.info(`Skipping PR #${pr.number} by ${author} (public member of ${trustedOrg}).`); + return; + } + // Repeating a qualifier narrows the search instead of widening it, + // hence the explicit disjunction. + const trustedMerges = await orNull(withRetries('Counting merged PRs in trusted organisations', () => + github.rest.search.issuesAndPullRequests({ + q: `author:${author} type:pr is:merged` + + ` (${TRUSTED_ORGS.map(org => `org:${org}`).join(' OR ')})`, + advanced_search: 'true', per_page: 1, + }).then(response => response.data.total_count), + )); + if (trustedMerges >= MIN_TRUSTED_MERGES) { + core.info(`Skipping PR #${pr.number} by ${author}` + + ` (${trustedMerges} PR(s) merged into ${TRUSTED_ORGS.join(', ')}).`); + return; + } + + const opened = new Date(pr.created_at); + const daysBefore = date => (opened - new Date(date)) / 86400000; + + // Signal 1: pull requests closed unmerged elsewhere, recently. + const search = await orNull(withRetries('Searching for PRs by the author', () => + github.rest.search.issuesAndPullRequests({ + q: `author:${author} type:pr`, advanced_search: 'true', + sort: 'created', order: 'desc', per_page: 100, + }).then(response => response.data), + )); + let rejections = null; + if (search) { + rejections = search.items.filter(item => { + const itemOwner = item.repository_url.split('/repos/')[1].split('/')[0].toLowerCase(); + return itemOwner !== author.toLowerCase() + && item.state === 'closed' && !item.pull_request?.merged_at + && daysBefore(item.created_at) >= 0 + && daysBefore(item.created_at) <= REJECTION_WINDOW_DAYS; + }).map(item => item.html_url); + } + + // Signal 2: how their recent comments across GitHub read. + const events = []; + for (let page = 1; page <= EVENT_PAGES; page++) { + const batch = await orNull(withRetries(`Reading public events page ${page}`, () => + github.rest.activity.listPublicEventsForUser({ + username: author, per_page: 100, page, + }).then(response => response.data), + )); + if (!batch?.length) break; + events.push(...batch); + if (batch.length < 100) break; + } + const comments = events + .filter(event => ['IssueCommentEvent', 'PullRequestReviewCommentEvent'].includes(event.type)) + .map(event => event.payload?.comment?.body) + .filter(Boolean); + + // Signal 3: how many unrelated projects they open pull requests + // against in a single week. Breadth rather than volume: a focused + // contributor sends many pull requests to few repositories, while + // an unattended agent sprays a few across many. Taken from the + // event feed, which unlike search covers authors that search + // refuses to return. + const weeks = {}; + for (const event of events) { + if (event.type !== 'PullRequestEvent' || event.payload?.action !== 'opened') continue; + const name = event.repo?.name; + if (!name || name.toLowerCase().startsWith(`${author.toLowerCase()}/`)) continue; + const week = Math.floor(new Date(event.created_at) / (7 * 86400000)); + (weeks[week] ??= new Set()).add(name); + } + const breadth = events.length + ? Math.max(0, ...Object.values(weeks).map(repos => repos.size)) + : null; + const STRUCTURE = [/^\s*#{2,3}\s/m, /^\s*[-*]\s.+\n\s*[-*]\s/m, /\*\*[^*]+\*\*/, /```/]; + const ACKNOWLEDGEMENT = [ + /thanks for (the )?(review|feedback|pointing|catching|flagging|clarif)/i, + /you'?re (absolutely )?right/i, /great catch/i, /that makes sense/i, + /i'?ll (continue|investigate|update|submit|look into|make sure)/i, + /let me know (if|whether)/i, /happy to (update|adjust|revise|change)/i, + /i understand that/i, /thanks for your time/i, /just following up/i, + /hope (this|that) helps/i, /please let me know/i, /i'?ve (updated|addressed|fixed)/i, + ]; + let voice = null; + if (comments.length >= MIN_COMMENTS) { + const chars = comments.reduce((total, body) => total + body.length, 0); + const rate = patterns => comments.filter(body => patterns.some(re => re.test(body))).length / comments.length; + voice = { + comments: comments.length, + structure: rate(STRUCTURE), + acknowledgement: rate(ACKNOWLEDGEMENT), + emDashPerKChar: 1000 * comments.reduce((total, body) => total + (body.match(/—/g) || []).length, 0) / chars, + }; + } + + const reasons = []; + if (rejections && rejections.length >= MIN_REJECTIONS) { + reasons.push(`${rejections.length} PR(s) of theirs closed unmerged elsewhere in the last` + + ` ${REJECTION_WINDOW_DAYS} days: ${rejections.slice(0, 10).join(' ')}`); + } + if (voice && (voice.structure > VOICE.structure + || voice.emDashPerKChar > VOICE.emDashPerKChar + || voice.acknowledgement > VOICE.acknowledgement)) { + reasons.push(`comment style over ${voice.comments} recent comments:` + + ` ${(100 * voice.structure).toFixed(0)}% structured,` + + ` ${(100 * voice.acknowledgement).toFixed(0)}% stock acknowledgements,` + + ` ${voice.emDashPerKChar.toFixed(2)} em dashes per 1000 characters`); + } + if (breadth !== null && breadth > MAX_REPOS_PER_WEEK) { + reasons.push(`opened pull requests against ${breadth} unrelated repositories within a week`); + } + if (AGENT_BRANCH.test(pr.head?.ref || '')) { + reasons.push(`branch name carries an agent prefix: ${pr.head.ref}`); + } + + await core.summary + .addHeading(`PR #${pr.number} by ${author}`, 3) + .addList([ + rejections === null + ? 'recent rejections elsewhere: unmeasurable, the author cannot be searched' + : `recent rejections elsewhere: ${rejections.length}`, + voice === null + ? `comment style: unmeasurable, fewer than ${MIN_COMMENTS} recent comments found` + : `comment style: ${(100 * voice.structure).toFixed(0)}% structured,` + + ` ${(100 * voice.acknowledgement).toFixed(0)}% stock acknowledgements,` + + ` ${voice.emDashPerKChar.toFixed(2)} em dashes per 1000 characters` + + ` over ${voice.comments} comments`, + breadth === null + ? 'repositories per week: unmeasurable, no public events found' + : `repositories per week, at most: ${breadth}`, + `branch: ${pr.head?.ref ?? 'unknown'}`, + `verdict: ${reasons.length ? `labelled "${LABEL}"` : 'not labelled'}`, + ]) + .addRaw(reasons.length ? `\n${reasons.map(reason => `- ${reason}`).join('\n')}\n` : '') + .write(); + + if (!reasons.length) { + core.info(`Not labelling PR #${pr.number}.`); + return; + } + await withRetries('Adding the label', () => + github.rest.issues.addLabels({ owner, repo, issue_number: pr.number, labels: [LABEL] }), + ); + core.info(`Labelled PR #${pr.number}: ${reasons.join(' | ')}`); diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 22b8996b6..697647131 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,21 +1,50 @@ name: Publish + +permissions: + contents: read + on: push: tags: - '[0-9]+.[0-9]+.[0-9]+' +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: - publish: + build: + name: Build distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - python-version: 3.11 - - run: | - pip install --upgrade build twine + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - run: | + python -m pip install --upgrade build python -m build - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.6.4 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - password: ${{ secrets.PYPI_TOKEN }} + name: python-package-distributions + path: dist/ + + publish: + name: Upload release to PyPI + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/Scrapy + permissions: + id-token: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: python-package-distributions + path: dist/ + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 3044a1af3..9a09aa67d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,26 +1,77 @@ name: macOS -on: [push, pull_request] + +permissions: + contents: read + +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: + +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true jobs: tests: - runs-on: macos-11 + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) + runs-on: macos-latest + env: + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + include: + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.14" + env: + TOXENV: py + coverage: true + - python-version: "3.14" + env: + TOXENV: no-reactor steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + run: uv tool install mitmproxy + - name: Run tests - run: | - pip install -U tox - tox -e py + env: ${{ matrix.env }} + run: uvx --with tox-uv tox - name: Upload coverage report - run: bash <(curl -s https://codecov.io/bash) + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + report_type: test_results diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ff92a571..cd726a2fe 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,82 +1,138 @@ name: Ubuntu -on: [push, pull_request] + +permissions: + contents: read + +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: + +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: ubuntu-latest + env: + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: include: - - python-version: 3.9 - env: - TOXENV: py - python-version: "3.10" env: TOXENV: py - python-version: "3.11" env: TOXENV: py - - python-version: "3.11" - env: - TOXENV: asyncio - - python-version: pypy3.9 - env: - TOXENV: pypy3 - - # pinned deps - - python-version: 3.8.17 - env: - TOXENV: pinned - - python-version: 3.8.17 - env: - TOXENV: asyncio-pinned - - python-version: pypy3.8 - env: - TOXENV: pypy3-pinned - - python-version: 3.8.17 - env: - TOXENV: extra-deps-pinned - - python-version: 3.8.17 - env: - TOXENV: botocore-pinned - - - python-version: "3.11" - env: - TOXENV: extra-deps - - python-version: "3.11" - env: - TOXENV: botocore - - - python-version: "3.12.0-rc.2" + - python-version: "3.12" env: TOXENV: py - - python-version: "3.12.0-rc.2" + - python-version: "3.13" env: - TOXENV: asyncio - - python-version: "3.12.0-rc.2" + TOXENV: py + - python-version: "3.14" + env: + TOXENV: py + coverage: true + - python-version: "3.14" + env: + TOXENV: default-reactor + coverage: true + - python-version: "3.14" + env: + TOXENV: no-reactor + coverage: true + + # min deps + - python-version: "3.10.19" + env: + TOXENV: min + coverage: true + - python-version: "3.10.19" + env: + TOXENV: min-default-reactor + coverage: true + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 + env: + TOXENV: min-pypy3 + - python-version: "3.10.19" + env: + TOXENV: min-extra-deps + coverage: true + - python-version: "3.10.19" + env: + TOXENV: min-botocore + coverage: true + + - python-version: "3.14" env: TOXENV: extra-deps + coverage: true + - python-version: "3.14" + env: + TOXENV: no-reactor-extra-deps + coverage: true + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 + env: + TOXENV: pypy3-extra-deps + - python-version: "3.14" + env: + TOXENV: botocore + coverage: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || contains(matrix.python-version, '3.12.0') + if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + # mitmproxy has no PyPy wheels, so run it on CPython regardless of the + # interpreter under test. + run: uv tool install --python cpython mitmproxy + - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report - run: bash <(curl -s https://codecov.io/bash) + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + report_type: test_results diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index c8d1928d7..254e9395e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,42 +1,89 @@ name: Windows -on: [push, pull_request] + +permissions: + contents: read + +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: + +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: windows-latest + env: + PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: include: - - python-version: 3.8 - env: - TOXENV: windows-pinned - - python-version: 3.9 - env: - TOXENV: py - python-version: "3.10" env: TOXENV: py - - python-version: "3.10" - env: - TOXENV: asyncio - - python-version: "3.11" + - python-version: "3.14" env: TOXENV: py - - python-version: "3.11" + coverage: true + - python-version: "3.14" env: - TOXENV: asyncio + TOXENV: no-reactor + + # min deps + - python-version: "3.10.11" + env: + TOXENV: min + - python-version: "3.10.11" + env: + TOXENV: min-extra-deps + + - python-version: "3.14" + env: + TOXENV: extra-deps steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + run: uv tool install mitmproxy + - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox + + - name: Upload coverage report + if: ${{ matrix.coverage }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + report_type: test_results diff --git a/.gitignore b/.gitignore index 6c5c50e08..5e52ecf1e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,18 +3,21 @@ *.pyc _trial_temp* dropin.cache -docs/build +docs/_build *egg-info -.tox -venv -build -dist -.idea +.tox/ +venv/ +.venv/ +build/ +dist/ +.idea/ +.vscode/ htmlcov/ -.coverage .pytest_cache/ +.coverage .coverage.* coverage.* +*.junit.xml test-output.* .cache/ .mypy_cache/ @@ -25,4 +28,4 @@ test-output.* Thumbs.db # OSX miscellaneous -.DS_Store \ No newline at end of file +.DS_Store diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index f238bf7ea..000000000 --- a/.isort.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -profile = black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0cff5cc73..c2cb5056d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,24 +1,37 @@ +exclude: | + (?x)( + ^docs/_static| + ^docs/_tests| + ^tests/sample_data + ) repos: -- repo: https://github.com/PyCQA/bandit - rev: 1.7.5 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.20 hooks: - - id: bandit - args: [-r, -c, .bandit.yml] -- repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 -- repo: https://github.com/psf/black.git - rev: 23.9.1 - hooks: - - id: black -- repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort + - id: ruff-check + args: [ --fix ] + - id: ruff-format - repo: https://github.com/adamchainz/blacken-docs - rev: 1.16.0 + rev: 1.20.0 hooks: - id: blacken-docs additional_dependencies: - - black==23.9.1 + - black==26.5.1 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.2 + hooks: + - id: sphinx-lint +- repo: https://github.com/scrapy/sphinx-scrapy + rev: 0.8.10 + hooks: + - id: sphinx-scrapy +- repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.28.0 + hooks: + - id: zizmor + args: [--no-progress, --fix] diff --git a/.readthedocs.yml b/.readthedocs.yml index e71d34f3a..a2773dcf2 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,17 +1,10 @@ version: 2 -formats: all -sphinx: - configuration: docs/conf.py - fail_on_warning: true - build: - os: ubuntu-20.04 + os: ubuntu-24.04 tools: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.11" # Keep in sync with .github/workflows/checks.yml - -python: - install: - - requirements: docs/requirements.txt - - path: . + python: "3.14" + commands: + - pip install tox + - tox -e docs + - mkdir -p $READTHEDOCS_OUTPUT/html + - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..24a426d36 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,6 @@ +cff-version: 1.2.0 +message: If you use Scrapy in published research, please cite it as below. +title: Scrapy +authors: + - name: Scrapy contributors +url: https://scrapy.org diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index ae7db51fa..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,26 +0,0 @@ -include README.rst -include AUTHORS -include INSTALL -include LICENSE -include MANIFEST.in -include NEWS - -include scrapy/VERSION -include scrapy/mime.types - -include codecov.yml -include conftest.py -include pytest.ini -include requirements-*.txt -include tox.ini - -recursive-include scrapy/templates * -recursive-include scrapy license.txt -recursive-include docs * -prune docs/build - -recursive-include extras * -recursive-include bin * -recursive-include tests * - -global-exclude __pycache__ *.py[cod] diff --git a/README.rst b/README.rst index 1918850d6..651294add 100644 --- a/README.rst +++ b/README.rst @@ -1,113 +1,54 @@ -.. image:: https://scrapy.org/img/scrapylogo.png - :target: https://scrapy.org/ - -====== -Scrapy -====== +|logo| -.. image:: https://img.shields.io/pypi/v/Scrapy.svg - :target: https://pypi.python.org/pypi/Scrapy +.. |logo| image:: https://raw.githubusercontent.com/scrapy/scrapy/master/docs/_static/logo.svg + :target: https://scrapy.org + :alt: Scrapy + :width: 480px + +|version| |python_version| |tests| |coverage| |conda| |deepwiki| + +.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg + :target: https://pypi.org/pypi/Scrapy :alt: PyPI Version -.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg - :target: https://pypi.python.org/pypi/Scrapy +.. |python_version| image:: https://img.shields.io/pypi/pyversions/Scrapy.svg + :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu - :alt: Ubuntu +.. |tests| image:: https://img.shields.io/github/check-runs/scrapy/scrapy/master?label=tests + :target: https://github.com/scrapy/scrapy/actions?query=branch%3Amaster + :alt: Tests -.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - :alt: macOS - -.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg - :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows - :alt: Windows - -.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg - :target: https://pypi.python.org/pypi/Scrapy - :alt: Wheel Status - -.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg +.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report -.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg +.. |conda| image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version +.. |deepwiki| image:: https://deepwiki.com/badge.svg + :target: https://deepwiki.com/scrapy/scrapy + :alt: Ask DeepWiki -Overview -======== - -Scrapy is a fast high-level web crawling and web scraping framework, used to -crawl websites and extract structured data from their pages. It can be used for -a wide range of purposes, from data mining to monitoring and automated testing. - -Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other -contributors`_. +Scrapy_ is a web scraping framework to extract structured data from websites. +It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_ +(formerly Scrapinghub) and `many other contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapy: https://scrapy.org/ .. _Zyte: https://www.zyte.com/ -Check the Scrapy homepage at https://scrapy.org for more information, -including a list of features. - - -Requirements -============ - -* Python 3.8+ -* Works on Linux, Windows, macOS, BSD - -Install -======= - -The quick way: +Install with: .. code:: bash pip install scrapy -See the install section in the documentation at -https://docs.scrapy.org/en/latest/intro/install.html for more details. +And follow the documentation_ to learn how to use it. -Documentation -============= +.. _documentation: https://docs.scrapy.org/en/latest/ -Documentation is available online at https://docs.scrapy.org/ and in the ``docs`` -directory. +If you wish to contribute, see Contributing_. -Releases -======== - -You can check https://docs.scrapy.org/en/latest/news.html for the release notes. - -Community (blog, twitter, mail list, IRC) -========================================= - -See https://scrapy.org/community/ for details. - -Contributing -============ - -See https://docs.scrapy.org/en/master/contributing.html for details. - -Code of Conduct ---------------- - -Please note that this project is released with a Contributor `Code of Conduct `_. - -By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@zyte.com. - -Companies using Scrapy -====================== - -See https://scrapy.org/companies/ for a list. - -Commercial Support -================== - -See https://scrapy.org/support/ for details. +.. _Contributing: https://docs.scrapy.org/en/master/contributing.html diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..c1a5482c6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 2.17.x | :white_check_mark: | +| < 2.17.x | :x: | + +## Reporting a Vulnerability + +Please report the vulnerability using https://github.com/scrapy/scrapy/security/advisories/new. diff --git a/artwork/README.rst b/artwork/README.rst deleted file mode 100644 index c1880ef6c..000000000 --- a/artwork/README.rst +++ /dev/null @@ -1,20 +0,0 @@ -============== -Scrapy artwork -============== - -This folder contains the Scrapy artwork resources such as logos and fonts. - -scrapy-logo.jpg ---------------- - -The main Scrapy logo, in JPEG format. - -qlassik.zip ------------ - -The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font - -scrapy-blog.logo.xcf --------------------- - -The logo used in the Scrapy blog, in Gimp format. diff --git a/artwork/qlassik.zip b/artwork/qlassik.zip deleted file mode 100644 index 2885c06ef..000000000 Binary files a/artwork/qlassik.zip and /dev/null differ diff --git a/artwork/scrapy-blog-logo.xcf b/artwork/scrapy-blog-logo.xcf deleted file mode 100644 index 320102604..000000000 Binary files a/artwork/scrapy-blog-logo.xcf and /dev/null differ diff --git a/artwork/scrapy-logo.jpg b/artwork/scrapy-logo.jpg deleted file mode 100644 index 4315ef8e1..000000000 Binary files a/artwork/scrapy-logo.jpg and /dev/null differ diff --git a/conftest.py b/conftest.py index 2bfa46f5a..5a535c168 100644 --- a/conftest.py +++ b/conftest.py @@ -1,14 +1,21 @@ -import platform -import sys +from __future__ import annotations + +import os +from importlib.util import find_spec from pathlib import Path +from typing import TYPE_CHECKING import pytest -from twisted import version as twisted_version -from twisted.python.versions import Version from twisted.web.http import H2_ENABLED -from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactor import set_asyncio_event_loop_policy +from scrapy.utils.reactorless import install_reactor_import_hook from tests.keys import generate_keys +from tests.mockserver.http import MockServer +from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd + +if TYPE_CHECKING: + from collections.abc import Generator def _py_files(folder): @@ -16,19 +23,21 @@ def _py_files(folder): collect_ignore = [ - # not a test, but looks like a test - "scrapy/utils/testsite.py", - "tests/ftpserver.py", - "tests/mockserver.py", - "tests/pipelines.py", - "tests/spiders.py", - # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess + # may need extra deps + "docs/_ext", + # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerProcessSubprocess + *_py_files("tests/AsyncCrawlerProcess"), + # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerRunnerSubprocess + *_py_files("tests/AsyncCrawlerRunner"), + # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), - # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), ] -with Path("tests/ignores.txt").open(encoding="utf-8") as reader: +base_dir = Path(__file__).parent +ignore_file_path = base_dir / "tests" / "ignores.txt" +with ignore_file_path.open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != "#": @@ -42,63 +51,94 @@ if not H2_ENABLED: ) ) +if find_spec("httpx2") is None and find_spec("httpx") is None: + collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") -@pytest.fixture() -def chdir(tmpdir): - """Change to pytest-provided temporary directory""" - tmpdir.chdir() +if find_spec("pytest_codspeed") is None: + collect_ignore.append("tests/benchmarks") -def pytest_addoption(parser): +def pytest_addoption(parser, pluginmanager): + if pluginmanager.hasplugin("twisted"): + return + # add the full choice set so that pytest doesn't complain about invalid choices in some cases parser.addoption( "--reactor", - default="default", - choices=["default", "asyncio"], + default="none", + choices=["asyncio", "default", "none"], ) -@pytest.fixture(scope="class") -def reactor_pytest(request): - if not request.cls: - # doctests - return - request.cls.reactor_pytest = request.config.getoption("--reactor") - return request.cls.reactor_pytest +@pytest.fixture(scope="session") +def mockserver() -> Generator[MockServer]: + with MockServer() as mockserver: + yield mockserver -@pytest.fixture(autouse=True) -def only_asyncio(request, reactor_pytest): - if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": - pytest.skip("This test is only run with --reactor=asyncio") +@pytest.fixture # function scope because it modifies os.environ +def proxy_server( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Generator[str]: + kind = request.param + proxy = MitmProxy(mode="socks5" if kind == "socks5" else None) + url = proxy.start() + if kind == "https": + url = url.replace("http://", "https://") + monkeypatch.setenv("http_proxy", url) + monkeypatch.setenv("https_proxy", url) + + try: + yield kind + finally: + proxy.stop() -@pytest.fixture(autouse=True) -def only_not_asyncio(request, reactor_pytest): - if ( - request.node.get_closest_marker("only_not_asyncio") - and reactor_pytest == "asyncio" - ): - pytest.skip("This test is only run without --reactor=asyncio") - - -@pytest.fixture(autouse=True) -def requires_uvloop(request): - if not request.node.get_closest_marker("requires_uvloop"): - return - if sys.implementation.name == "pypy": - pytest.skip("uvloop does not support pypy properly") - if platform.system() == "Windows": - pytest.skip("uvloop does not support Windows") - if twisted_version == Version("twisted", 21, 2, 0): - pytest.skip("https://twistedmatrix.com/trac/ticket/10106") - if sys.version_info >= (3, 12): - pytest.skip("uvloop doesn't support Python 3.12 yet") +@pytest.fixture(scope="session") +def reactor_pytest(request) -> str: + return request.config.getoption("--reactor") def pytest_configure(config): if config.getoption("--reactor") == "asyncio": - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + # Needed on Windows to switch from proactor to selector for Twisted reactor compatibility. + # If we decide to run tests with both, we will need to add a new option and check it here. + set_asyncio_event_loop_policy() + elif config.getoption("--reactor") == "none": + install_reactor_import_hook() -# Generate localhost certificate files, needed by some tests -generate_keys() +def pytest_runtest_setup(item): + # Skip tests based on reactor markers + reactor = item.config.getoption("--reactor") + + if item.get_closest_marker("requires_reactor") and reactor == "none": + pytest.skip('This test is only run when the --reactor value is not "none"') + + if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}: + pytest.skip( + 'This test is only run when the --reactor value is "asyncio" (default) or "none"' + ) + + if item.get_closest_marker("only_not_asyncio") and reactor in {"asyncio", "none"}: + pytest.skip( + 'This test is only run when the --reactor value is not "asyncio" (default) or "none"' + ) + + # Skip tests requiring optional dependencies + optional_deps = [ + "uvloop", + "botocore", + "boto3", + ] + + for module in optional_deps: + if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None: + pytest.skip(f"{module} is not installed") + + if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None: + pytest.skip("mitmdump is not available") + + +# Generate localhost certificate files, needed by some tests (but only once if xdist is used) +if "PYTEST_XDIST_WORKER" not in os.environ: + generate_keys() diff --git a/docs/Makefile b/docs/Makefile index 48401bac8..ed8809902 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,96 +1,20 @@ -# -# Makefile for Scrapy documentation [based on Python documentation Makefile] -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -PYTHON = python -SPHINXOPTS = -PAPER = -SOURCES = -SHELL = /usr/bin/env bash - -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ - -D latex_elements.papersize=$(PAPER) \ - $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) - -.PHONY: help update build html htmlhelp clean +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = build +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " text to make plain text files" - @echo " changes to make an overview over all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " watch build HTML docs, open in browser and watch for changes" + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -build-dirs: - mkdir -p build/$(BUILDER) build/doctrees +.PHONY: help Makefile -build: build-dirs - sphinx-build $(ALLSPHINXOPTS) - @echo - -build-ignore-errors: build-dirs - -sphinx-build $(ALLSPHINXOPTS) - @echo - - -html: BUILDER = html -html: build - @echo "Build finished. The HTML pages are in build/html." - -htmlhelp: BUILDER = htmlhelp -htmlhelp: build - @echo "Build finished; now you can run HTML Help Workshop with the" \ - "build/htmlhelp/pydoc.hhp project file." - -latex: BUILDER = latex -latex: build - @echo "Build finished; the LaTeX files are in build/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -text: BUILDER = text -text: build - @echo "Build finished; the text files are in build/text." - -changes: BUILDER = changes -changes: build - @echo "The overview file is in build/changes." - -linkcheck: BUILDER = linkcheck -linkcheck: build - @echo "Link check complete; look for any errors in the above output " \ - "or in build/$(BUILDER)/output.txt" - -linkfix: BUILDER = linkcheck -linkfix: build-ignore-errors - $(PYTHON) utils/linkfix.py - @echo "Fixing redirecting links in docs has finished; check all " \ - "replacements before committing them" - -doctest: BUILDER = doctest -doctest: build - @echo "Testing of doctests in the sources finished, look at the " \ - "results in build/doctest/output.txt" - -pydoc-topics: BUILDER = pydoc-topics -pydoc-topics: build - @echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \ - "into the Lib/ directory" - -coverage: BUILDER = coverage -coverage: build - -htmlview: html - $(PYTHON) -c "import webbrowser; from pathlib import Path; \ - webbrowser.open(Path('build/html/index.html').resolve().as_uri())" - -clean: - -rm -rf build/* - -watch: htmlview - watchmedo shell-command -p '*.rst' -c 'make html' -R -D +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.rst b/docs/README.rst deleted file mode 100644 index 36dd5aea4..000000000 --- a/docs/README.rst +++ /dev/null @@ -1,68 +0,0 @@ -:orphan: - -====================================== -Scrapy documentation quick start guide -====================================== - -This file provides a quick guide on how to compile the Scrapy documentation. - - -Setup the environment ---------------------- - -To compile the documentation you need Sphinx Python library. To install it -and all its dependencies run the following command from this dir - -:: - - pip install -r requirements.txt - - -Compile the documentation -------------------------- - -To compile the documentation (to classic HTML output) run the following command -from this dir:: - - make html - -Documentation will be generated (in HTML format) inside the ``build/html`` dir. - - -View the documentation ----------------------- - -To view the documentation run the following command:: - - make htmlview - -This command will fire up your default browser and open the main page of your -(previously generated) HTML documentation. - - -Start over ----------- - -To clean up all generated documentation files and start from scratch run:: - - make clean - -Keep in mind that this command won't touch any documentation source files. - - -Recreating documentation on the fly ------------------------------------ - -There is a way to recreate the doc automatically when you make changes, you -need to install watchdog (``pip install watchdog``) and then use:: - - make watch - -Alternative method using tox ----------------------------- - -To compile the documentation to HTML run the following command:: - - tox -e docs - -Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir. diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index c23a89089..edb91bfb9 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,62 +1,67 @@ +# pylint: disable=import-error +from collections.abc import Sequence from operator import itemgetter +from typing import Any, TypedDict from docutils import nodes +from docutils.nodes import Element, General, Node, document from docutils.parsers.rst import Directive -from docutils.parsers.rst.roles import set_classes +from sphinx.application import Sphinx from sphinx.util.nodes import make_refnode -class settingslist_node(nodes.General, nodes.Element): +class SettingData(TypedDict): + docname: str + setting_name: str + refid: str + + +class SettingslistNode(General, Element): pass class SettingsListDirective(Directive): - def run(self): - return [settingslist_node("")] + def run(self) -> Sequence[Node]: + return [SettingslistNode()] -def is_setting_index(node): - if node.tagname == "index" and node["entries"]: +def is_setting_index(node: Node) -> bool: + if node.tagname == "index" and node["entries"]: # type: ignore[index,attr-defined] # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] - entry_type, info, refid = node["entries"][0][:3] + entry_type, info, _ = node["entries"][0][:3] # type: ignore[index] return entry_type == "pair" and info.endswith("; setting") return False -def get_setting_target(node): - # target nodes are placed next to the node in the doc tree - return node.parent[node.parent.index(node) + 1] - - -def get_setting_name_and_refid(node): +def get_setting_name_and_refid(node: Node) -> tuple[str, str]: """Extract setting name from directive index node""" - entry_type, info, refid = node["entries"][0][:3] + _, info, refid = node["entries"][0][:3] # type: ignore[index] return info.replace("; setting", ""), refid -def collect_scrapy_settings_refs(app, doctree): +def collect_scrapy_settings_refs(app: Sphinx, doctree: document) -> None: env = app.builder.env if not hasattr(env, "scrapy_all_settings"): - env.scrapy_all_settings = [] - - for node in doctree.traverse(is_setting_index): - targetnode = get_setting_target(node) - assert isinstance(targetnode, nodes.target), "Next node is not a target" + emptyList: list[SettingData] = [] + env.scrapy_all_settings = emptyList # type: ignore[attr-defined] + for node in doctree.findall(is_setting_index): setting_name, refid = get_setting_name_and_refid(node) - env.scrapy_all_settings.append( - { - "docname": env.docname, - "setting_name": setting_name, - "refid": refid, - } + env.scrapy_all_settings.append( # type: ignore[attr-defined] + SettingData( + docname=env.docname, + setting_name=setting_name, + refid=refid, + ) ) -def make_setting_element(setting_data, app, fromdocname): +def make_setting_element( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> Any: refnode = make_refnode( app.builder, fromdocname, @@ -72,77 +77,106 @@ def make_setting_element(setting_data, app, fromdocname): return item -def replace_settingslist_nodes(app, doctree, fromdocname): +def make_setting_markdown_item( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> str: + uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"]) + if uri.startswith("#"): + target = f"#{setting_data['refid']}" + else: + target = f"{uri}#{setting_data['refid']}" + return f"* [{setting_data['setting_name']}]({target})" + + +def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]: + return [ + d + for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] + if fromdocname != d["docname"] + ] + + +def replace_settingslist_nodes( + app: Sphinx, doctree: document, fromdocname: str +) -> None: env = app.builder.env - for node in doctree.traverse(settingslist_node): + for node in doctree.findall(SettingslistNode): settings_list = nodes.bullet_list() settings_list.extend( [ make_setting_element(d, app, fromdocname) - for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) - if fromdocname != d["docname"] + for d in _iter_sorted_settings(env, fromdocname) ] ) node.replace_self(settings_list) -def setup(app): - app.add_crossref_type( - directivename="setting", - rolename="setting", - indextemplate="pair: %s; setting", - ) - app.add_crossref_type( - directivename="signal", - rolename="signal", - indextemplate="pair: %s; signal", - ) - app.add_crossref_type( - directivename="command", - rolename="command", - indextemplate="pair: %s; command", - ) - app.add_crossref_type( - directivename="reqmeta", - rolename="reqmeta", - indextemplate="pair: %s; reqmeta", - ) +def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None: + builder = translator.builder + env = builder.env + fromdocname = getattr(builder, "current_doc_name", env.docname) + lines = [ + make_setting_markdown_item(setting_data, builder.app, fromdocname) + for setting_data in _iter_sorted_settings(env, fromdocname) + ] + if lines: + translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2) + raise nodes.SkipNode + + +def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None: + return None + + +def source_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/blob/master/" + text + node = nodes.reference(rawtext, text, refuri=ref, **options) + return [node], [] + + +def issue_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/issues/" + text + node = nodes.reference(rawtext, "issue " + text, refuri=ref) + return [node], [] + + +def commit_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/commit/" + text + node = nodes.reference(rawtext, "commit " + text, refuri=ref) + return [node], [] + + +def rev_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "http://hg.scrapy.org/scrapy/changeset/" + text + node = nodes.reference(rawtext, "r" + text, refuri=ref) + return [node], [] + + +def setup(app: Sphinx) -> dict[str, Any]: app.add_role("source", source_role) app.add_role("commit", commit_role) app.add_role("issue", issue_role) app.add_role("rev", rev_role) - app.add_node(settingslist_node) + app.add_node( + SettingslistNode, + markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown), + singlemarkdown=( + visit_settingslist_node_markdown, + depart_settingslist_node_markdown, + ), + ) app.add_directive("settingslist", SettingsListDirective) app.connect("doctree-read", collect_scrapy_settings_refs) app.connect("doctree-resolved", replace_settingslist_nodes) - - -def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/blob/master/" + text - set_classes(options) - node = nodes.reference(rawtext, text, refuri=ref, **options) - return [node], [] - - -def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/issues/" + text - set_classes(options) - node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options) - return [node], [] - - -def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/commit/" + text - set_classes(options) - node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options) - return [node], [] - - -def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "http://hg.scrapy.org/scrapy/changeset/" + text - set_classes(options) - node = nodes.reference(rawtext, "r" + text, refuri=ref, **options) - return [node], [] + return {"parallel_read_safe": True} diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py new file mode 100644 index 000000000..e342e92cf --- /dev/null +++ b/docs/_ext/scrapyfixautodoc.py @@ -0,0 +1,21 @@ +""" +Must be included after 'sphinx.ext.autodoc'. Fixes unwanted 'alias of' behavior. +https://github.com/sphinx-doc/sphinx/issues/4422 +""" + +from typing import Any + +# pylint: disable=import-error +from sphinx.application import Sphinx + + +def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool: + if not skip: + # autodoc was generating the text "alias of" for the following members + return name in {"default_item_class", "default_selector_class"} + return skip + + +def setup(app: Sphinx) -> dict[str, Any]: + app.connect("autodoc-skip-member", maybe_skip_member) + return {"parallel_read_safe": True} diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 64f16939c..1c2859deb 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -7,4 +7,50 @@ } .rst-content dl p + ol, .rst-content dl p + ul { margin-top: -6px; /* Compensates margin-top: 12px of p */ -} \ No newline at end of file +} + +/*override some styles in +sphinx-rtd-dark-mode/static/dark_mode_css/general.css*/ +.theme-switcher { + right: 0.4em !important; + top: 0.6em !important; + -webkit-box-shadow: 0px 3px 14px 4px rgba(0, 0, 0, 0.30) !important; + box-shadow: 0px 3px 14px 4px rgba(0, 0, 0, 0.30) !important; + height: 2em !important; + width: 2em !important; +} + +/*place the toggle button for dark mode +at the bottom right corner on small screens*/ +@media (max-width: 768px) { + .theme-switcher { + right: 0.4em !important; + bottom: 2.6em !important; + top: auto !important; + } +} + +/*persist blue color at the top left used in +default rtd theme*/ +html[data-theme="dark"] .wy-side-nav-search, +html[data-theme="dark"] .wy-nav-top { + background-color: #1d577d !important; +} + +/*all the styles below used to present +API objects nicely in dark mode*/ +html[data-theme="dark"] .sig.sig-object { + border-left-color: #3e4446 !important; + background-color: #202325 !important +} + +html[data-theme="dark"] .sig-name, +html[data-theme="dark"] .sig-prename, +html[data-theme="dark"] .property, +html[data-theme="dark"] .sig-param, +html[data-theme="dark"] .sig-paren, +html[data-theme="dark"] .sig-return-icon, +html[data-theme="dark"] .sig-return-typehint, +html[data-theme="dark"] .optional { + color: #e8e6e3 !important +} diff --git a/docs/_static/logo.svg b/docs/_static/logo.svg new file mode 100644 index 000000000..04b2d18a7 --- /dev/null +++ b/docs/_static/logo.svg @@ -0,0 +1 @@ + diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 000000000..29394799b --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,23 @@ +{% extends "!layout.html" %} + +{# Overridden to include a link to scrapy.org, not just to the docs root #} +{%- block sidebartitle %} + +{# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #} +{# the master_doc variable was renamed to root_doc in Sphinx 4 (master_doc still exists in later Sphinx versions) #} +{%- set _logo_url = logo_url|default(pathto('_static/' + (logo or ""), 1)) %} +{%- set _root_doc = root_doc|default(master_doc) %} +scrapy.org / docs + +{%- if READTHEDOCS or DEBUG %} + {%- if theme_version_selector or theme_language_selector %} +
+
+
+
+ {%- endif %} +{%- endif %} + +{%- include "searchbox.html" %} + +{%- endblock %} diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html index 71aff8847..d1cfd9020 100644 --- a/docs/_tests/quotes.html +++ b/docs/_tests/quotes.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
@@ -273,7 +273,7 @@ Quotes by: GoodReads.com

- Made with ❤ by Scrapinghub + Made with ❤ by Zyte

diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html index 71aff8847..d1cfd9020 100644 --- a/docs/_tests/quotes1.html +++ b/docs/_tests/quotes1.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
@@ -273,7 +273,7 @@ Quotes by: GoodReads.com

- Made with ❤ by Scrapinghub + Made with ❤ by Zyte

diff --git a/docs/conf.py b/docs/conf.py index 38ca81932..1b41adaad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,16 +1,11 @@ -# Scrapy documentation build configuration file, created by -# sphinx-quickstart on Mon Nov 24 12:02:52 2008. +# Configuration file for the Sphinx documentation builder. # -# This file is execfile()d with the current directory set to its containing dir. -# -# The contents of this file are pickled, so don't put values in the namespace -# that aren't pickleable (module imports are okay, they're removed automatically). -# -# All configuration values have a default; values that are commented out -# serve to show the default. +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html +import os import sys -from datetime import datetime +from collections.abc import Sequence from pathlib import Path # If your extensions are in another directory, add it here. If the directory @@ -19,36 +14,28 @@ sys.path.append(str(Path(__file__).parent / "_ext")) sys.path.insert(0, str(Path(__file__).parent.parent)) -# General configuration -# --------------------- +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "Scrapy" +project_copyright = "Scrapy developers" +author = "Scrapy developers" + + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - "hoverxref.extension", "notfound.extension", "scrapydocs", - "sphinx.ext.autodoc", + "sphinx_scrapy", + "scrapyfixautodoc", # Must be after "sphinx.ext.autodoc" "sphinx.ext.coverage", - "sphinx.ext.intersphinx", - "sphinx.ext.viewcode", + "sphinx_rtd_dark_mode", ] -# Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = "Scrapy" -copyright = f"2008–{datetime.now().year}, Scrapy developers" +exclude_patterns = ["build", "Thumbs.db", ".DS_Store"] # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -64,138 +51,34 @@ except ImportError: version = "" release = "" -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -language = "en" - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of documents that shouldn't be included in the build. -# unused_docs = [] - -exclude_patterns = ["build"] - -# List of directories, relative to source directory, that shouldn't be searched -# for source files. -exclude_trees = [".build"] - -# The reST default role (used for this markup: `text`) to use for all documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# List of Sphinx warnings that will not be raised suppress_warnings = ["epub.unknown_project_files"] -# Options for HTML output -# ----------------------- +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. html_theme = "sphinx_rtd_theme" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# Add path to the RTD explicitly to robustify builds (otherwise might -# fail in a clean Debian build env) -import sphinx_rtd_theme - -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - -# The style sheet to use for HTML and HTML Help pages. A file of that name -# must exist either in Sphinx' static/ path, or in one of the custom paths -# given in html_static_path. -# html_style = 'scrapydoc.css' - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. html_last_updated_fmt = "%b %d, %Y" -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_use_modindex = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, the reST sources are included in the HTML build as _sources/. -html_copy_source = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = '' - -# Output file base name for HTML help builder. -htmlhelp_basename = "Scrapydoc" - html_css_files = [ "custom.css", ] +html_context = { + "display_github": True, + "github_user": "scrapy", + "github_repo": "scrapy", + "github_version": "master", + "conf_py_path": "/docs/", +} -# Options for LaTeX output -# ------------------------ +# Set canonical URL from the Read the Docs Domain +html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") -# The paper size ('letter' or 'a4'). -# latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -# latex_font_size = '10pt' +# -- Options for LaTeX output ------------------------------------------------ +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-latex-output # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -203,38 +86,22 @@ latex_documents = [ ("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"), ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False +# -- Options for the linkcheck builder --------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-the-linkcheck-builder -# Additional stuff for the LaTeX preamble. -# latex_preamble = '' - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_use_modindex = True - - -# Options for the linkcheck builder -# --------------------------------- - -# A list of regular expressions that match URIs that should not be checked when -# doing a linkcheck build. linkcheck_ignore = [ - "http://localhost:\d+", + r"http://localhost:\d+", "http://hg.scrapy.org", - "http://directory.google.com/", + r"https://github.com/scrapy/scrapy/commit/\w+", + r"https://github.com/scrapy/scrapy/issues/\d+", ] +linkcheck_anchors_ignore_for_url = ["https://github.com/pyca/cryptography/issues/2692"] + +# -- Options for the Coverage extension -------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/coverage.html#configuration -# Options for the Coverage extension -# ---------------------------------- coverage_ignore_pyobjects = [ # Contract’s add_pre_hook and add_post_hook are not documented because # they should be transparent to contract developers, for whom pre_hook and @@ -254,6 +121,10 @@ coverage_ignore_pyobjects = [ # Base classes of downloader middlewares are implementation details that # are not meant for users. r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware", + # The interface methods of duplicate request filtering classes are already + # covered in the interface documentation part of the DUPEFILTER_CLASS + # setting documentation. + r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_crawler|request_seen|open|close|log)$", # Private exception used by the command-line interface implementation. r"^scrapy\.exceptions\.UsageError", # Methods of BaseItemExporter subclasses are only documented in @@ -270,52 +141,33 @@ coverage_ignore_pyobjects = [ r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor", ] +# -- Options for the autodoc extension ---------------------------------------- +autodoc_member_order = "bysource" -# Options for the InterSphinx extension -# ------------------------------------- +# -- Options for the InterSphinx extension ----------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration -intersphinx_mapping = { - "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/stable", None), - "cryptography": ("https://cryptography.io/en/latest/", None), - "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), - "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), - "pytest": ("https://docs.pytest.org/en/latest", None), - "python": ("https://docs.python.org/3", None), - "sphinx": ("https://www.sphinx-doc.org/en/master", None), - "tox": ("https://tox.wiki/en/latest/", None), - "twisted": ("https://docs.twisted.org/en/stable/", None), - "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), - "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), -} -intersphinx_disabled_reftypes = [] +intersphinx_disabled_reftypes: Sequence[str] = [] +# sphinx-scrapy --------------------------------------------------------------- -# Options for sphinx-hoverxref options -# ------------------------------------ +scrapy_intersphinx_enable = [ + "attrs", + "coverage", + "cryptography", + "cssselect", + "form2request", + "itemloaders", + "parsel", + "pytest", + "pypug", + "scrapy-lint", + "sphinx", + "tox", + "twisted", + "twistedapi", + "w3lib", +] -hoverxref_auto_ref = True -hoverxref_role_types = { - "class": "tooltip", - "command": "tooltip", - "confval": "tooltip", - "hoverxref": "tooltip", - "mod": "tooltip", - "ref": "tooltip", - "reqmeta": "tooltip", - "setting": "tooltip", - "signal": "tooltip", -} -hoverxref_roles = ["command", "reqmeta", "setting", "signal"] - - -def setup(app): - app.connect("autodoc-skip-member", maybe_skip_member) - - -def maybe_skip_member(app, what, name, obj, skip, options): - if not skip: - # autodocs was generating a text "alias of" for the following members - # https://github.com/sphinx-doc/sphinx/issues/4422 - return name in {"default_item_class", "default_selector_class"} - return skip +# -- Other options ------------------------------------------------------------ +default_dark_mode = False diff --git a/docs/contributing.rst b/docs/contributing.rst index 2b3249601..6d2e08fe8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,8 +6,13 @@ Contributing to Scrapy .. important:: - Double check that you are reading the most recent version of this document at - https://docs.scrapy.org/en/master/contributing.html + Double check that you are reading the most recent version of this document + at https://docs.scrapy.org/en/master/contributing.html + + By participating in this project you agree to abide by the terms of our + `Code of Conduct + `_. Please + report unacceptable behavior to opensource@zyte.com. There are many ways to contribute to Scrapy. Here are some of them: @@ -74,18 +79,81 @@ guidelines when you're going to report a new bug. .. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve +.. _find-work: + +Finding work +============ + +If you have decided to make a contribution to Scrapy, but you do not know what +to contribute, you have a few options to find pending work: + +- Check out the `contribution GitHub page`_, which lists open issues tagged + as **good first issue**. + + .. _contribution GitHub page: https://github.com/scrapy/scrapy/contribute + + There are also `help wanted issues`_ but mind that some may require + familiarity with the Scrapy code base. You can also target any other issue + provided it is not tagged as **discuss**. + +- If you enjoy writing documentation, there are `documentation issues`_ as + well, but mind that some may require familiarity with the Scrapy code base + as well. + + .. _documentation issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3Adocs+ + +- If you enjoy :ref:`writing automated tests `, you can work on + increasing our `test coverage`_. + +- If you enjoy code cleanup, we welcome fixes for issues detected by our + static analysis tools. See ``pyproject.toml`` for silenced issues that may + need addressing. + + Mind that some issues we do not aim to address at all, and usually include + a comment on them explaining the reason; not to confuse with comments that + state what the issue is about, for non-descriptive issue codes. + +If you have found an issue, make sure you read the entire issue thread before +you ask questions. That includes related issues and pull requests that show up +in the issue thread when the issue is mentioned elsewhere. + +We do not assign issues, and you do not need to announce that you are going to +start working on an issue either. If you want to work on an issue, just go +ahead and :ref:`write a patch for it `. + +Do not discard an issue simply because there is an open pull request for it. +Check if open pull requests are active first. And even if some are active, if +you think you can build a better implementation, feel free to create a pull +request with your approach. + +If you decide to work on something without an open issue, please: + +- Do not create an issue to work on code coverage or code cleanup, create a + pull request directly. + +- Do not create both an issue and a pull request right away. Either open an + issue first to get feedback on whether or not the issue is worth + addressing, and create a pull request later only if the feedback from the + team is positive, or create only a pull request, if you think a discussion + will be easier over your code. + +- Do not add docstrings for the sake of adding docstrings, or only to address + silenced Ruff issues. We expect docstrings to exist only when they add + something significant to readers, such as explaining something that is not + easier to understand from reading the corresponding code, summarizing a + long, hard-to-read implementation, providing context about calling code, or + indicating purposely uncaught exceptions from called code. + +- Do not add tests that use as much mocking as possible just to touch a given + line of code and hence improve line coverage. While we do aim to maximize + test coverage, tests should be written for real scenarios, with minimum + mocking. We usually prefer end-to-end tests. + .. _writing-patches: Writing patches =============== -Scrapy has a list of `good first issues`_ and `help wanted issues`_ that you -can work on. These issues are a great way to get started with contributing to -Scrapy. If you're new to the codebase, you may want to focus on documentation -or testing-related issues, as they are always useful and can help you get -more familiar with the project. You can also check Scrapy's `test coverage`_ -to see which areas may benefit from more tests. - The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged. Well-written patches should: @@ -131,6 +199,14 @@ Remember to explain what was fixed or the new functionality (what it is, why it's needed, etc). The more info you include, the easier will be for core developers to understand and accept your patch. +If your pull request aims to resolve an open issue, `link it accordingly +`__, +e.g.: + +.. code-block:: none + + Resolves #123 + You can also discuss the new functionality (or bug fix) before creating the patch, but it's always good to have a patch ready to illustrate your arguments and show that you have put some additional thought into the subject. A good @@ -154,7 +230,7 @@ by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE`` (replace 'upstream' with a remote name for scrapy repository, ``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE`` with a name of the branch you want to create locally). -See also: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. +See also: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. When writing GitHub pull requests, try to keep titles short but descriptive. E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" @@ -175,15 +251,15 @@ Coding style Please follow these coding conventions when writing code for inclusion in Scrapy: -* We use `black `_ for code formatting. +* We use `Ruff `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e black``. + run Ruff manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. - See https://help.github.com/en/github/using-git/setting-your-username-in-git for - setup instructions. + See https://docs.github.com/en/get-started/git-basics/setting-your-username-in-git + for setup instructions. .. _scrapy-pre-commit: @@ -242,13 +318,15 @@ Documentation about deprecated features must be removed as those features are deprecated, so that new readers do not run into it. New deprecations and deprecation removals are documented in the :ref:`release notes `. +.. _write-tests: Tests ===== -Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires -:doc:`tox `. +Tests are implemented using pytest_. Running tests requires :doc:`tox +`. + +.. _pytest: https://pytest.org .. _running-tests: @@ -294,6 +372,21 @@ To see coverage report install :doc:`coverage ` see output of ``coverage --help`` for more options like html or xml report. +Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a +fully featured proxy server; they are skipped when one cannot be found +(``mitmproxy`` is intentionally not a test dependency that would be installed +into test venvs, as that sometimes leads to various dependency conflicts). +To run these tests, make ``mitmdump`` available in one of these ways: + +* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with + pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``); + +* have uv_ installed, in which case the tests will run + ``uvx --from mitmproxy mitmdump``; + +* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump`` + executable. + Writing tests ------------- @@ -313,13 +406,14 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users -.. _Scrapy subreddit: https://reddit.com/r/scrapy -.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS +.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/ .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues -.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ -.. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request +.. _PEP 257: https://peps.python.org/pep-0257/ +.. _pull request: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist -.. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy +.. _mitmproxy: https://mitmproxy.org/ +.. _pipx: https://pipx.pypa.io/ +.. _uv: https://docs.astral.sh/uv/ diff --git a/docs/faq.rst b/docs/faq.rst index 20dd814df..80658a5bf 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -23,7 +23,7 @@ comparing `jinja2`_ to `Django`_. .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: https://lxml.de/ -.. _jinja2: https://palletsprojects.com/p/jinja/ +.. _jinja2: https://palletsprojects.com/projects/jinja/ .. _Django: https://www.djangoproject.com/ Can I use Scrapy with BeautifulSoup? @@ -82,44 +82,35 @@ to steal from us! Does Scrapy work with HTTP proxies? ----------------------------------- -Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP -Proxy downloader middleware. See +Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader +middleware. See :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`. +Does Scrapy work with SOCKS proxies? +------------------------------------ + +Yes, when using +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the +handler documentation. + How can I scrape an item with attributes in different pages? ------------------------------------------------------------ -See :ref:`topics-request-response-ref-request-callback-arguments`. +See :ref:`callback-data`. How can I simulate a user login in my spider? --------------------------------------------- See :ref:`topics-request-response-ref-request-userlogin`. + .. _faq-bfo-dfo: Does Scrapy crawl in breadth-first or depth-first order? -------------------------------------------------------- -By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which -basically means that it crawls in `DFO order`_. This order is more convenient -in most cases. - -If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings: - -.. code-block:: python - - DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" - SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" - -While pending requests are below the configured values of -:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent -concurrently. As a result, the first few requests of a crawl rarely follow the -desired order. Lowering those settings to ``1`` enforces the desired order, but -it significantly slows down the crawl as a whole. +:ref:`DFO by default, but other orders are possible `. My Scrapy crawler has memory leaks. What can I do? @@ -138,39 +129,36 @@ See previous question. How can I prevent memory errors due to many allowed domains? ------------------------------------------------------------ -If you have a spider with a long list of -:attr:`~scrapy.Spider.allowed_domains` (e.g. 50,000+), consider -replacing the default -:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` spider middleware -with a :ref:`custom spider middleware ` that requires -less memory. For example: +If you have a spider with a long list of :attr:`~scrapy.Spider.allowed_domains` +(e.g. 50,000+), consider replacing the default +:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` downloader +middleware with a :ref:`custom downloader middleware +` that requires less memory. For example: - If your domain names are similar enough, use your own regular expression - instead joining the strings in - :attr:`~scrapy.Spider.allowed_domains` into a complex regular - expression. + instead of joining the strings in :attr:`~scrapy.Spider.allowed_domains` into + a complex regular expression. -- If you can `meet the installation requirements`_, use pyre2_ instead of +- If you can meet the installation requirements, use pyre2_ instead of Python’s re_ to compile your URL-filtering regular expression. See :issue:`1908`. -See also other suggestions at `StackOverflow`_. +See also `other suggestions at StackOverflow +`__. .. note:: Remember to disable - :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable - your custom implementation: + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` when you + enable your custom implementation: .. code-block:: python - SPIDER_MIDDLEWARES = { - "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, - "myproject.middlewares.CustomOffsiteMiddleware": 500, + DOWNLOADER_MIDDLEWARES = { + "scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None, + "myproject.middlewares.CustomOffsiteMiddleware": 50, } -.. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation .. _pyre2: https://github.com/andreasvc/pyre2 -.. _re: https://docs.python.org/library/re.html -.. _StackOverflow: https://stackoverflow.com/q/36440681/939364 +.. _re: https://docs.python.org/3/library/re.html Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- @@ -206,12 +194,10 @@ I get "Filtered offsite request" messages. How can I fix them? Those messages (logged with ``DEBUG`` level) don't necessarily mean there is a problem, so you may not need to fix them. -Those messages are thrown by the Offsite Spider Middleware, which is a spider -middleware (enabled by default) whose purpose is to filter out requests to -domains outside the ones covered by the spider. - -For more info see: -:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware`. +Those messages are thrown by +:class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, which is a +downloader middleware (enabled by default) whose purpose is to filter out +requests to domains outside the ones covered by the spider. What is the recommended way to deploy a Scrapy crawler in production? --------------------------------------------------------------------- @@ -234,21 +220,15 @@ the :ref:`topics-signals-ref` to know which ones. What does the response status code 999 mean? -------------------------------------------- -999 is a custom response status code used by Yahoo sites to throttle requests. +999 is a custom response status code used by some sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or -higher) in your spider: +higher) for the affected domains, with the :setting:`DOWNLOAD_SLOTS` setting: .. code-block:: python - from scrapy.spiders import CrawlSpider - - - class MySpider(CrawlSpider): - name = "myspider" - - download_delay = 2 - - # [ ... rest of the spider code ... ] + DOWNLOAD_SLOTS = { + "example.com": {"delay": 2}, + } Or by setting a global download delay in your project with the :setting:`DOWNLOAD_DELAY` setting. @@ -273,7 +253,7 @@ To dump into a CSV file:: scrapy crawl myspider -O items.csv -To dump into a XML file:: +To dump into an XML file:: scrapy crawl myspider -O items.xml @@ -286,7 +266,7 @@ The ``__VIEWSTATE`` parameter is used in sites built with ASP.NET/VB.NET. For more info on how it works see `this page`_. Also, here's an `example spider`_ which scrapes one of these sites. -.. _this page: https://metacpan.org/pod/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm +.. _this page: https://metacpan.org/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/view/lib/HTML/TreeBuilderX/ASP_NET.pm .. _example spider: https://github.com/AmbientLighter/rpn-fas/blob/master/fas/spiders/rnp.py What's the best way to parse big XML/CSV data feeds? @@ -297,9 +277,14 @@ build the DOM of the entire feed in memory, and this can be quite slow and consume a lot of memory. In order to avoid parsing all the entire feed at once in memory, you can use -the functions ``xmliter`` and ``csviter`` from ``scrapy.utils.iterators`` -module. In fact, this is what the feed spiders (see :ref:`topics-spiders`) use -under the cover. +the :func:`~scrapy.utils.iterators.xmliter_lxml` and +:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what +:class:`~scrapy.spiders.XMLFeedSpider` and +:class:`~scrapy.spiders.CSVFeedSpider` use. + +.. autofunction:: scrapy.utils.iterators.xmliter_lxml + +.. autofunction:: scrapy.utils.iterators.csviter Does Scrapy manage cookies automatically? ----------------------------------------- @@ -341,8 +326,8 @@ section of the site (which varies each time). In that case, the credentials to log in would be settings, while the url of the section to scrape would be a spider argument. -I'm scraping a XML document and my XPath selector doesn't return any items --------------------------------------------------------------------------- +I'm scraping an XML document and my XPath selector doesn't return any items +--------------------------------------------------------------------------- You may need to remove namespaces. See :ref:`removing-namespaces`. @@ -362,21 +347,27 @@ method for this purpose. For example: from copy import deepcopy - from itemadapter import is_item, ItemAdapter + from itemadapter import ItemAdapter + from scrapy import Request class MultiplyItemsMiddleware: - def process_spider_output(self, response, result, spider): - for item in result: - if is_item(item): - adapter = ItemAdapter(item) - for _ in range(adapter["multiply_by"]): - yield deepcopy(item) + def process_spider_output(self, response, result): + for item_or_request in result: + if isinstance(item_or_request, Request): + yield item_or_request + continue + adapter = ItemAdapter(item_or_request) + for _ in range(adapter["multiply_by"]): + yield deepcopy(item_or_request) Does Scrapy support IPv6 addresses? ----------------------------------- -Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. +Yes, but when using +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or +:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to +set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. Note that by doing so, you lose the ability to set a specific timeout for DNS requests (the value of the :setting:`DNS_TIMEOUT` setting is ignored). @@ -387,8 +378,9 @@ How to deal with ``: filedescriptor out of range in select() ---------------------------------------------------------------------------------------------- This issue `has been reported`_ to appear when running broad crawls in macOS, where the default -Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a -different reactor is possible by using the :setting:`TWISTED_REACTOR` setting. +Twisted reactor was :class:`twisted.internet.selectreactor.SelectReactor` at that time. +If you have switched to this reactor using the :setting:`TWISTED_REACTOR` setting you can switch +to a different one in the same way. .. _faq-stop-response-download: @@ -405,6 +397,22 @@ or :class:`~scrapy.signals.headers_received` signals and raising a :ref:`topics-stop-response-download` topic for additional information and examples. +.. _faq-blank-request: + +How can I make a blank request? +------------------------------- + +.. code-block:: python + + from scrapy import Request + + blank_request = Request("data:,") + +In this case, the URL is set to a data URI scheme. Data URLs allow you to include data +inline within web pages, similar to external resources. The "data:" scheme with an empty +content (",") essentially creates a request to a data URL without any specific content. + + Running ``runspider`` I get ``error: No spider found in file: `` -------------------------------------------------------------------------- @@ -415,9 +423,5 @@ See :issue:`2680`. .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 -.. _Python standard library modules: https://docs.python.org/py-modindex.html +.. _Python standard library modules: https://docs.python.org/3/py-modindex.html .. _Python package: https://pypi.org/ -.. _user agents: https://en.wikipedia.org/wiki/User_agent -.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) -.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/index.rst b/docs/index.rst index 8798aebd1..688cab81b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ Having trouble? We'd like to help! * Ask or search questions in `StackOverflow using the scrapy tag`_. * Ask or search questions in the `Scrapy subreddit`_. * Search for questions on the archives of the `scrapy-users mailing list`_. -* Ask a question in the `#scrapy IRC channel`_, +* Ask a question in the `#scrapy IRC channel`_. * Report bugs with Scrapy in our `issue tracker`_. * Join the Discord community `Scrapy Discord`_. @@ -33,7 +33,7 @@ Having trouble? We'd like to help! .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues -.. _Scrapy Discord: https://discord.gg/mv3yErfpvq +.. _Scrapy Discord: https://discord.com/invite/mv3yErfpvq First steps @@ -91,15 +91,15 @@ Basic concepts :doc:`topics/selectors` Extract the data from web pages using XPath. -:doc:`topics/shell` - Test your extraction code in an interactive environment. - :doc:`topics/items` Define the data you want to scrape. :doc:`topics/loaders` Populate your items with the extracted data. +:doc:`topics/shell` + Test your extraction code in an interactive environment. + :doc:`topics/item-pipeline` Post-process and store your scraped data. @@ -128,18 +128,14 @@ Built-in services topics/logging topics/stats - topics/email topics/telnetconsole :doc:`topics/logging` - Learn how to use Python's builtin logging on Scrapy. + Learn how to use Python's built-in logging on Scrapy. :doc:`topics/stats` Collect statistics about your scraping crawler. -:doc:`topics/email` - Send email notifications when certain events occur. - :doc:`topics/telnetconsole` Inspect a running crawler using a built-in Python console. @@ -155,6 +151,7 @@ Solving specific problems topics/debug topics/contracts topics/practices + topics/security topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -179,6 +176,10 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. +:doc:`topics/security` + Understand the security implications of Scrapy defaults and how to harden + them. + :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. @@ -229,6 +230,7 @@ Extending Scrapy topics/signals topics/scheduler topics/exporters + topics/download-handlers topics/components topics/api @@ -257,6 +259,9 @@ Extending Scrapy :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). +:doc:`topics/download-handlers` + Customize how requests are downloaded or add support for new URL schemes. + :doc:`topics/components` Learn the common API and some good practices when building custom Scrapy components. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index c90c1d2bf..cba8c15a1 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.8+, either the CPython implementation (default) or +Scrapy requires Python 3.10+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: @@ -37,7 +37,7 @@ Note that sometimes this may require solving compilation issues for some Scrapy dependencies depending on your operating system, so be sure to check the :ref:`intro-install-platform-notes`. -For more detailed and platform specifics instructions, as well as +For more detailed and platform-specific instructions, as well as troubleshooting information, read on. @@ -89,6 +89,56 @@ just like any other Python package. (See :ref:`platform-specific guides ` below for non-Python dependencies that you may need to install beforehand). +.. _extras: + +Optional extras +=============== + +Scrapy provides optional :ref:`extras ` +that install additional dependencies to enable specific features. To install +Scrapy with one or more extras, list them in square brackets: + +.. code-block:: console + + pip install scrapy[s3,images] + +The following extras are available: + +.. list-table:: + :header-rows: 1 + + * - Extra + - Provides + * - ``bpython`` + - :ref:`bpython shell ` + * - ``brotli`` + - :ref:`Brotli response decompression ` + * - ``gcs`` + - :ref:`Google Cloud Storage ` for + :ref:`feed exports ` and + :ref:`media pipelines ` + * - ``httpx`` + - :ref:`httpx-handler`, including its HTTP/2 and SOCKS proxy support + * - ``images`` + - :ref:`Images pipeline ` + * - ``ipython`` + - :ref:`IPython shell ` + * - ``ptpython`` + - :ref:`ptpython shell ` + * - ``robotparser`` + - :ref:`Robotexclusionrulesparser robots.txt parsing ` + * - ``s3`` + - :ref:`Amazon S3 ` storage for + :ref:`feed exports `, + :ref:`media pipelines `, and + :ref:`S3 downloads ` + * - ``twisted-http2`` + - :ref:`twisted-http2-handler` + * - ``uvloop`` + - `uvloop `_ event loop + * - ``zstd`` + - :ref:`Zstandard response decompression ` + .. _intro-install-platform-notes: @@ -101,7 +151,7 @@ Windows ------- Though it's possible to install Scrapy on Windows using pip, we recommend you -to install `Anaconda`_ or `Miniconda`_ and use the package from the +install `Anaconda`_ or `Miniconda`_ and use the package from the `conda-forge`_ channel, which will avoid most installation issues. Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: @@ -111,7 +161,7 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: To install Scrapy on Windows using ``pip``: .. warning:: - This installation method requires “Microsoft Visual C++” for installing some + This installation method requires “Microsoft Visual C++” for installing some Scrapy dependencies, which demands significantly more disk space than Anaconda. #. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. @@ -123,7 +173,7 @@ To install Scrapy on Windows using ``pip``: #. Check the installation details and make sure following packages are selected as optional components: * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) - + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) #. Install the Visual Studio Build Tools. @@ -141,7 +191,7 @@ But it should support older versions of Ubuntu too, like Ubuntu 14.04, albeit with potential issues with TLS connections. **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are -typically too old and slow to catch up with latest Scrapy. +typically too old and slow to catch up with the latest Scrapy release. To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install @@ -170,7 +220,7 @@ macOS Building Scrapy's dependencies requires the presence of a C compiler and development headers. On macOS this is typically provided by Apple’s Xcode -development tools. To install the Xcode command line tools open a terminal +development tools. To install the Xcode command-line tools, open a terminal window and run:: xcode-select --install @@ -200,11 +250,6 @@ solutions: brew install python - * Latest versions of python have ``pip`` bundled with them so you won't need - to install it separately. If this is not the case, upgrade python:: - - brew update; brew upgrade python - * *(Optional)* :ref:`Install Scrapy inside a Python virtual environment `. @@ -235,8 +280,8 @@ Installing Scrapy with PyPy on Windows is not tested. You can check that Scrapy is installed correctly by running ``scrapy bench``. If this command gives errors such as ``TypeError: ... got 2 unexpected keyword arguments``, this means -that setuptools was unable to pick up one PyPy-specific dependency. -To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. +that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run +``pip install 'PyPyDispatcher>=2.1.0'``. .. _intro-install-troubleshooting: @@ -268,18 +313,16 @@ reinstall Twisted with the :code:`tls` extra option:: For details, see `Issue #2473 `_. .. _Python: https://www.python.org/ -.. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: https://lxml.de/index.html .. _parsel: https://pypi.org/project/parsel/ .. _w3lib: https://pypi.org/project/w3lib/ -.. _twisted: https://twistedmatrix.com/trac/ +.. _twisted: https://twisted.org/ .. _cryptography: https://cryptography.io/en/latest/ .. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ -.. _setuptools: https://pypi.python.org/pypi/setuptools +.. _setuptools: https://pypi.org/pypi/setuptools .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Anaconda: https://docs.anaconda.com/anaconda/ +.. _Anaconda: https://www.anaconda.com/docs/main .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html -.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio .. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 542760b4f..5937c5860 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -44,13 +44,13 @@ https://quotes.toscrape.com, following the pagination: if next_page is not None: yield response.follow(next_page, self.parse) -Put this in a text file, name it to something like ``quotes_spider.py`` +Put this in a text file, name it something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: scrapy runspider quotes_spider.py -o quotes.jsonl When this finishes you will have in the ``quotes.jsonl`` file a list of the -quotes in JSON Lines format, containing text and author, looking like this:: +quotes in JSON Lines format, containing the text and author, which will look like this:: {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} @@ -65,34 +65,35 @@ When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked fo Spider definition inside it and ran it through its crawler engine. The crawl started by making requests to the URLs defined in the ``start_urls`` -attribute (in this case, only the URL for quotes in *humor* category) +attribute (in this case, only the URL for quotes in the *humor* category) and called the default callback method ``parse``, passing the response object as an argument. In the ``parse`` callback, we loop through the quote elements using a CSS Selector, yield a Python dict with the extracted quote text and author, look for a link to the next page and schedule another request using the same ``parse`` method as callback. -Here you notice one of the main advantages about Scrapy: requests are +Here you will notice one of the main advantages of Scrapy: requests are :ref:`scheduled and processed asynchronously `. This means that Scrapy doesn't need to wait for a request to be finished and processed, it can send another request or do other things in the meantime. This -also means that other requests can keep going even if some request fails or an +also means that other requests can keep going even if a request fails or an error happens while handling it. While this enables you to do very fast crawls (sending multiple concurrent requests at the same time, in a fault-tolerant way) Scrapy also gives you control over the politeness of the crawl through :ref:`a few settings `. You can do things like setting a download delay between -each request, limiting amount of concurrent requests per domain or per IP, and +each request, limiting the amount of concurrent requests per domain, and even :ref:`using an auto-throttling extension ` that tries -to figure out these automatically. +to figure these settings out automatically. .. note:: This is using :ref:`feed exports ` to generate the - JSON file, you can easily change the export format (XML or CSV, for example) or the - storage backend (FTP or `Amazon S3`_, for example). You can also write an - :ref:`item pipeline ` to store the items in a database. + JSON Lines file, you can easily change the export format (XML or CSV, for + example) or the storage backend (FTP or `Amazon S3`_, for example). You can + also write an :ref:`item pipeline ` to store the + items in a database. .. _topics-whatelse: @@ -106,10 +107,10 @@ scraping easy and efficient, such as: * Built-in support for :ref:`selecting and extracting ` data from HTML/XML sources using extended CSS selectors and XPath expressions, - with helper methods to extract using regular expressions. + with helper methods for extraction using regular expressions. * An :ref:`interactive shell console ` (IPython aware) for trying - out the CSS and XPath expressions to scrape data, very useful when writing or + out the CSS and XPath expressions to scrape data, which is very useful when writing or debugging your spiders. * Built-in support for :ref:`generating feed exports ` in @@ -124,7 +125,7 @@ scraping easy and efficient, such as: well-defined API (middlewares, :ref:`extensions `, and :ref:`pipelines `). -* Wide range of built-in extensions and middlewares for handling: +* A wide range of built-in extensions and middlewares for handling: - cookies and session handling - HTTP features like compression, authentication, caching @@ -150,8 +151,8 @@ The next steps for you are to :ref:`install Scrapy `, a full-blown Scrapy project and `join the community`_. Thanks for your interest! -.. _join the community: https://scrapy.org/community/ +.. _join the community: https://www.scrapy.org/community .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping -.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html +.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/welcome/ecs .. _Amazon S3: https://aws.amazon.com/s3/ .. _Sitemaps: https://www.sitemaps.org/index.html diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 19a76fc16..eaf492c95 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -18,11 +18,11 @@ This tutorial will walk you through these tasks: 4. Changing spider to recursively follow links 5. Using spider arguments -Scrapy is written in Python_. If you're new to the language you might want to -start by getting an idea of what the language is like, to get the most out of -Scrapy. +Scrapy is written in Python_. The more you learn about Python, the more you +can get out of Scrapy. -If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource. +If you're already familiar with other languages and want to learn Python quickly, the +`Python Tutorial`_ is a good resource. If you're new to programming and want to start with Python, the following books may be useful to you: @@ -76,10 +76,9 @@ This will create a ``tutorial`` directory with the following contents:: Our first Spider ================ -Spiders are classes that you define and that Scrapy uses to scrape information -from a website (or a group of websites). They must subclass -:class:`~scrapy.Spider` and define the initial requests to make, -optionally how to follow links in the pages, and how to parse the downloaded +Spiders are classes that you define and that Scrapy uses to scrape information from a website +(or a group of websites). They must subclass :class:`~scrapy.Spider` and define the initial +requests to be made, and optionally, how to follow links in pages and parse the downloaded page content to extract data. This is the code for our first Spider. Save it in a file named @@ -95,7 +94,7 @@ This is the code for our first Spider. Save it in a file named class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): urls = [ "https://quotes.toscrape.com/page/1/", "https://quotes.toscrape.com/page/2/", @@ -117,10 +116,10 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.Spider.start_requests`: must return an iterable of - Requests (you can return a list of requests or write a generator function) - which the Spider will begin to crawl from. Subsequent requests will be - generated successively from these initial requests. +* :meth:`~scrapy.Spider.start`: must be an asynchronous generator that + yields requests (and, optionally, items) for the spider to start crawling. + Subsequent requests will be generated successively from these initial + requests. * :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter @@ -138,7 +137,7 @@ To put our spider to work, go to the project's top level directory and run:: scrapy crawl quotes -This command runs the spider with name ``quotes`` that we've just added, that +This command runs the spider named ``quotes`` that we've just added, that will send some requests for the ``quotes.toscrape.com`` domain. You will get an output similar to this:: @@ -165,21 +164,22 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects -returned by the ``start_requests`` method of the Spider. Upon receiving a -response for each one, it instantiates :class:`~scrapy.http.Response` objects -and calls the callback method associated with the request (in this case, the -``parse`` method) passing the response as argument. +Scrapy sends the first :class:`scrapy.Request ` objects yielded +by the :meth:`~scrapy.Spider.start` spider method. Upon receiving a +response for each one, Scrapy calls the callback method associated with the +request (in this case, the ``parse`` method) with a +:class:`~scrapy.http.Response` object. -A shortcut to the start_requests method ---------------------------------------- -Instead of implementing a :meth:`~scrapy.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.Spider.start_urls` class attribute -with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider. +A shortcut to the ``start`` method +---------------------------------- + +Instead of implementing a :meth:`~scrapy.Spider.start` method that yields +:class:`~scrapy.Request` objects from URLs, you can define a +:attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This +list will then be used by the default implementation of +:meth:`~scrapy.Spider.start` to create the initial requests for your +spider. .. code-block:: python @@ -217,8 +217,8 @@ using the :ref:`Scrapy shell `. Run:: .. note:: - Remember to always enclose urls in quotes when running Scrapy shell from - command-line, otherwise urls containing arguments (i.e. ``&`` character) + Remember to always enclose URLs in quotes when running Scrapy shell from the + command line, otherwise URLs containing arguments (i.e. ``&`` character) will not work. On Windows, use double quotes instead:: @@ -257,7 +257,7 @@ object: The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of :class:`~scrapy.Selector` objects that wrap around XML/HTML elements -and allow you to run further queries to fine-grain the selection or extract the +and allow you to run further queries to refine the selection or extract the data. To extract the text from the title above, you can do: @@ -293,7 +293,7 @@ As an alternative, you could've written: >>> response.css("title::text")[0].get() 'Quotes to Scrape' -Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will raise an :exc:`IndexError` exception if there are no results: .. code-block:: pycon @@ -303,8 +303,8 @@ raise an :exc:`IndexError` exception if there are no results: ... IndexError: list index out of range -You might want to use ``.get()`` directly on the -:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` if there are no results: .. code-block:: pycon @@ -354,12 +354,12 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions: XPath expressions are very powerful, and are the foundation of Scrapy Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You -can see that if you read closely the text representation of the selector -objects in the shell. +can see that if you read the text representation of the selector +objects in the shell closely. While perhaps not as popular as CSS selectors, XPath expressions offer more power because besides navigating the structure, it can also look at the -content. Using XPath, you're able to select things like: *select the link +content. Using XPath, you're able to select things like: *the link that contains the text "Next Page"*. This makes XPath very fitting to the task of scraping, and we encourage you to learn XPath even if you already know how to construct CSS selectors, it will make scraping much easier. @@ -370,7 +370,7 @@ recommend `this tutorial to learn XPath through examples `_, and `this tutorial to learn "how to think in XPath" `_. -.. _XPath: https://www.w3.org/TR/xpath/all/ +.. _XPath: https://www.w3.org/TR/xpath-10/ .. _CSS: https://www.w3.org/TR/selectors Extracting quotes and authors @@ -422,7 +422,7 @@ variable, so that we can run our CSS selectors directly on a particular quote: >>> quote = response.css("div.quote")[0] -Now, let's extract ``text``, ``author`` and the ``tags`` from that quote +Now, let's extract the ``text``, ``author`` and ``tags`` from that quote using the ``quote`` object we just created: .. code-block:: pycon @@ -448,7 +448,7 @@ to get all of them: from sys import version_info Having figured out how to extract each bit, we can now iterate over all the -quotes elements and put them together into a Python dictionary: +quote elements and put them together into a Python dictionary: .. code-block:: pycon @@ -465,8 +465,8 @@ quotes elements and put them together into a Python dictionary: Extracting data in our spider ----------------------------- -Let's get back to our spider. Until now, it doesn't extract any data in -particular, just saves the whole HTML page to a local file. Let's integrate the +Let's get back to our spider. Until now, it hasn't extracted any data in +particular, just saving the whole HTML page to a local file. Let's integrate the extraction logic above into our spider. A Scrapy spider typically generates many dictionaries containing the data @@ -493,7 +493,15 @@ in the callback, as you can see below: "tags": quote.css("div.tags a.tag::text").getall(), } -If you run this spider, it will output the extracted data with the log:: +To run this spider, exit the scrapy shell by entering:: + + quit() + +Then, run:: + + scrapy crawl quotes + +Now, it should output the extracted data with the log:: 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} @@ -521,8 +529,8 @@ using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jsonl -The `JSON Lines`_ format is useful because it's stream-like, you can easily -append new records to it. It doesn't have the same problem of JSON when you run +The `JSON Lines`_ format is useful because it's stream-like, so you can easily +append new records to it. It doesn't have the same problem as JSON when you run twice. Also, as each record is a separate line, you can process big files without having to fit everything in memory, there are tools like `JQ`_ to help do that at the command-line. @@ -534,7 +542,7 @@ for Item Pipelines has been set up for you when the project is created, in ``tutorial/pipelines.py``. Though you don't need to implement any item pipelines if you just want to store the scraped items. -.. _JSON Lines: http://jsonlines.org +.. _JSON Lines: https://jsonlines.org .. _JQ: https://stedolan.github.io/jq @@ -547,7 +555,7 @@ from https://quotes.toscrape.com, you want quotes from all the pages in the webs Now that you know how to extract data from pages, let's see how to follow links from them. -First thing is to extract the link to the page we want to follow. Examining +The first thing to do is extract the link to the page we want to follow. Examining our page, we can see there is a link to the next page with the following markup: @@ -581,7 +589,7 @@ There is also an ``attrib`` property available >>> response.css("li.next a").attrib["href"] '/page/2/' -Let's see now our spider modified to recursively follow the link to the next +Now let's see our spider, modified to recursively follow the link to the next page, extracting data from it: .. code-block:: python @@ -748,8 +756,8 @@ Another interesting thing this spider demonstrates is that, even if there are many quotes from the same author, we don't need to worry about visiting the same author page multiple times. By default, Scrapy filters out duplicated requests to URLs already visited, avoiding the problem of hitting servers too -much because of a programming mistake. This can be configured by the setting -:setting:`DUPEFILTER_CLASS`. +much because of a programming mistake. This can be configured in the +:setting:`DUPEFILTER_CLASS` setting. Hopefully by now you have a good understanding of how to use the mechanism of following links and callbacks with Scrapy. @@ -761,7 +769,7 @@ crawlers on top of it. Also, a common pattern is to build an item with data from more than one page, using a :ref:`trick to pass additional data to the callbacks -`. +`. Using spider arguments @@ -787,7 +795,7 @@ with a specific tag, building the URL based on the argument: class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): url = "https://quotes.toscrape.com/" tag = getattr(self, "tag", None) if tag is not None: @@ -816,12 +824,12 @@ Next steps ========== This tutorial covered only the basics of Scrapy, but there's a lot of other -features not mentioned here. Check the :ref:`topics-whatelse` section in +features not mentioned here. Check the :ref:`topics-whatelse` section in the :ref:`intro-overview` chapter for a quick overview of the most important ones. You can continue from the section :ref:`section-basics` to know more about the command-line tool, spiders, selectors and other things the tutorial hasn't covered like -modeling the scraped data. If you prefer to play with an example project, check +modeling the scraped data. If you'd prefer to play with an example project, check the :ref:`intro-examples` section. .. _JSON: https://en.wikipedia.org/wiki/JSON diff --git a/docs/news.rst b/docs/news.rst index 9e758f05d..a5cc6c723 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,22 +3,3415 @@ Release notes ============= -.. _release-2.11.0: - -Scrapy 2.11.0 (to be released) ------------------------------- +Scrapy VERSION (unreleased) +--------------------------- Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- The following runtime usage of zope.interface_ interfaces is removed: + + - :class:`~scrapy.spiderloader.SpiderLoader` and + :class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked + as implementing the ``ISpiderLoader`` interface. + + - :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the + configured spider loader implements the ``ISpiderLoader`` interface. + + - :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`, + :class:`~scrapy.extensions.feedexport.FileFeedStorage` and + :class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer + marked as implementing the ``IFeedStorage`` interface. + + (:issue:`6585`, :issue:`7731`) + +.. _release-2.17.0: + +Scrapy 2.17.0 (2026-07-07) +-------------------------- + +Highlights: + +- Security bug fixes + +- HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler`` + +- Improved settings for changing allowed TLS versions + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- ``s3://`` requests now use HTTPS by default, instead of plaintext HTTP. + + Previously, :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` + sent signed S3 requests over plaintext HTTP unless + ``request.meta["is_secure"]`` was set to a true value, exposing the request + path, the AWS ``Authorization`` header, the ``X-Amz-Security-Token`` header + (when using temporary credentials), and the response contents to network + attackers, who could also tamper with responses. See the `76g3-c3x4-crvx`_ + security advisory for details. + + To restore the previous behavior for a given request, set + ``request.meta["is_secure"]`` to ``False``. + + .. _76g3-c3x4-crvx: https://github.com/scrapy/scrapy/security/advisories/GHSA-76g3-c3x4-crvx + +Deprecations +~~~~~~~~~~~~ + +- The ``DOWNLOADER_CLIENT_TLS_METHOD`` setting is deprecated. You should use + the :setting:`DOWNLOAD_TLS_MIN_VERSION` and/or + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings instead if you want to change + the TLS method selection. + (:issue:`3288`, :issue:`6546`) + +- The following spider attributes are deprecated in favor of settings: + + - ``http_user`` (use :setting:`HTTPAUTH_USER`) + + - ``http_pass`` (use :setting:`HTTPAUTH_PASS`) + + - ``http_auth_domain`` (use :setting:`HTTPAUTH_DOMAIN`) + + (:issue:`7590`) + +- The ``scrapy.commands.ScrapyCommand.help()`` method is deprecated. It was + never called by Scrapy. + (:issue:`7626`, :issue:`7633`) + +- The following TLS-related functions and constants, intended for internal + use, are deprecated: + + - ``scrapy.core.downloader.tls.METHOD_TLS`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv10`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv11`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv12`` + + - ``scrapy.core.downloader.tls.openssl_methods`` + + - ``scrapy.core.downloader.tls.DEFAULT_CIPHERS`` + + - ``scrapy.utils.ssl.ffi_buf_to_string()`` + + - ``scrapy.utils.ssl.get_temp_key_info()`` + + - ``scrapy.utils.ssl.x509name_to_string()`` + + (:issue:`6546`, :issue:`7619`, :issue:`7665`) + +- The ``CRAWLSPIDER_FOLLOW_LINKS`` setting is deprecated. You can set + ``follow=False`` in your rules to achieve the same effect. + (:issue:`7592`) + +- Instantiating + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + without a ``crawler`` argument is deprecated. + (:issue:`7655`) + +- Instantiating + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` without a + ``settings`` argument is deprecated. + (:issue:`7664`) + +New features +~~~~~~~~~~~~ + +- Added support for HTTP/2 requests to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. It + requires setting the new :setting:`HTTPX_HTTP2_ENABLED` setting to + ``True``. + (:issue:`7575`) + +- Added support for SOCKS proxies to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + (:issue:`747`, :issue:`7575`) + +- Added :setting:`DOWNLOAD_TLS_MIN_VERSION` and + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings as replacements for the + ``DOWNLOADER_CLIENT_TLS_METHOD`` setting (which is now deprecated). + Compared to the old setting, they support specifying a range of allowed + versions and support newer TLS versions. + (:issue:`4821`, :issue:`6546`) + +- Added :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS` and + :setting:`HTTPAUTH_DOMAIN` settings and :reqmeta:`http_user`, + :reqmeta:`http_pass` and :reqmeta:`http_auth_domain` meta keys as more + flexible ways to set HTTP authentication data. + (:issue:`7590`) + +- Added a :reqmeta:`verbatim_url` meta key that can be set to ``True`` to + skip request URL canonicalization. + (:issue:`7473`) + +- Added ``deny_tags`` and ``deny_attrs`` arguments to :class:`LinkExtractor + `. + (:issue:`6321`, :issue:`7679`) + +- :attr:`scrapy.Item.fields` now returns the fields in the definition order + instead of the alphabetical one. + (:issue:`7015`, :issue:`7694`) + +- Added a :setting:`RETRY_GIVE_UP_LOG_LEVEL` setting, a + :reqmeta:`give_up_log_level` meta key and a ``give_up_log_level`` argument + of the + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function that + allow changing the log level of the message logged when the retry limit has + been reached. + (:issue:`4622`, :issue:`5297`, :issue:`7567`) + +- It's now possible to set :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` to + ``None`` to use the default ciphers of the underlying TLS implementation. + (:issue:`7499`, :issue:`7665`) + +Improvements +~~~~~~~~~~~~ + +- :class:`~scrapy.FormRequest` is no longer deprecated, only its + ``from_response()`` method is still deprecated. + (:issue:`7561`, :issue:`7671`) + +- Switched the item definition in the default project template from a + :class:`scrapy.item.Item` to a dataclass. + (:issue:`7493`, :issue:`7513`) + +- Fixed deprecation warnings with pyOpenSSL 26.3.0. + (:issue:`7619`) + +- Removed the runtime warnings for :attr:`Spider.allowed_domains + ` containing URLs or domains with ports + instead of just domains and for spider classes having a ``start_url`` + attribute instead of :class:`~scrapy.spiders.Spider.start_urls`. Please use + :doc:`scrapy-lint ` to find mistakes in your spider code + instead. + (:issue:`4421`, :issue:`7627`) + +- :func:`scrapy.utils.test.get_crawler` now disables + :setting:`TELNETCONSOLE_ENABLED` by default. + (:issue:`7644`) + +- Other code refactoring and improvements. + (:issue:`7409`, :issue:`7593`, :issue:`7594`, :issue:`7611`, :issue:`7649`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` no + longer ignores proxy credentials for redirected or retried requests. + (:issue:`7601`, :issue:`7630`) + +- :class:`~scrapy.extensions.feedexport.GCSFeedStorage` now closes the + temporary file after the upload. + (:issue:`7546`) + +- Fixed ``scrapy shell `` running a full spider crawl when there is a + spider for the requested URL. This bug was introduced in Scrapy 2.13.0. + (:issue:`7552`, :issue:`7557`) + +- The :setting:`IMAGES_STORE_S3_ACL` and :setting:`IMAGES_STORE_GCS_ACL` + settings are no longer ignored. This bug was introduced in Scrapy 2.12.0. + (:issue:`7597`, :issue:`7614`) + +- :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` now closes + the connection after making the request. + (:issue:`7602`, :issue:`7667`) + +- Removed the deprecated ``spider`` argument from the pipeline defined in the + default project template. + (:issue:`7676`) + +- Fixed ``scrapy genspider --edit`` not working. + (:issue:`7260`, :issue:`7683`) + +- When a :class:`~scrapy.crawler.Crawler` instance is passed to + :meth:`AsyncCrawlerRunner.create_crawler() + ` or + :meth:`CrawlerRunner.create_crawler() + `, settings from both classes + are now merged, previously only the settings from the + :class:`~scrapy.crawler.Crawler` instance were used. + (:issue:`1280`, :issue:`7647`) + +- Fixed several issues with cookie handling in + :func:`scrapy.utils.request.request_to_curl`. + (:issue:`7603`, :issue:`7675`, :issue:`7684`) + +- Fixed :class:`scrapy.resolver.CachingThreadedResolver` not disabling the + cache when :setting:`DNSCACHE_ENABLED` is set to ``False``. + (:issue:`7663`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` not removing comments + when looking for the ```` tag. + (:issue:`7506`) + +- Fixed checking for deprecated methods in custom :setting:`ITEM_PROCESSOR` + implementations. + (:issue:`7589`) + +- Fixed :func:`scrapy.utils.url.strip_url` corrupting some URLs with + credentials. + (:issue:`7604`, :issue:`7605`) + +- :func:`scrapy.utils.misc.rel_has_nofollow` now ignores the case when + looking for "nofollow" strings. + (:issue:`7632`) + +- Fixed an exception in :class:`scrapy.utils.sitemap.Sitemap` when parsing + some malformed sitemaps. + (:issue:`7686`, :issue:`7687`) + +Documentation +~~~~~~~~~~~~~ + +- Mentioned :doc:`scrapy-lint ` in the docs. + (:issue:`4421`, :issue:`7627`) + +- Added the docs about :ref:`security considerations `. + (:issue:`7389`, :issue:`7678`) + +- Improved the :ref:`item pipeline docs `. + (:issue:`2350`, :issue:`7676`) + +- Documented which stats are collected by + :class:`~scrapy.extensions.corestats.CoreStats`. + (:issue:`7421`) + +- Switched documentation examples from using :class:`scrapy.item.Item` to + using dataclasses. + (:issue:`7493`, :issue:`7513`) + +- Added feature comparison tables to the :ref:`download handler + ` docs. + (:issue:`7575`) + +- Improved the docs for :ref:`logging settings `. + (:issue:`6909`, :issue:`7668`) + +- Documented a way to :ref:`improve startup time and memory usage + ` by using :setting:`SPIDER_MODULES`. + (:issue:`7576`, :issue:`7600`) + +- Clarified handling of the ``type`` argument of :class:`~scrapy.Selector`. + (:issue:`7704`) + +- Other documentation improvements and fixes. + (:issue:`4954`, + :issue:`6120`, + :issue:`7286`, + :issue:`7564`, + :issue:`7573`, + :issue:`7598`, + :issue:`7599`, + :issue:`7698`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Fixed deprecation warnings with pytest 9.1.0. + (:issue:`7621`) + +- Type hints improvements and fixes. + (:issue:`6958`, :issue:`7586`) + +- CI and test improvements and fixes. + (:issue:`5954`, + :issue:`7002`, + :issue:`7017`, + :issue:`7247`, + :issue:`7508`, + :issue:`7545`, + :issue:`7566`, + :issue:`7574`, + :issue:`7585`, + :issue:`7595`, + :issue:`7608`, + :issue:`7610`, + :issue:`7612`, + :issue:`7616`, + :issue:`7625`, + :issue:`7637`, + :issue:`7639`, + :issue:`7640`, + :issue:`7641`, + :issue:`7642`, + :issue:`7643`, + :issue:`7644`, + :issue:`7645`, + :issue:`7646`, + :issue:`7654`, + :issue:`7655`, + :issue:`7664`, + :issue:`7672`, + :issue:`7677`, + :issue:`7680`, + :issue:`7682`, + :issue:`7692`) + +.. _release-2.16.0: + +Scrapy 2.16.0 (2026-05-19) +-------------------------- + +Highlights: + +- Official support for Python 3.14 + +- Support for Twisted 26.4.0+ + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Increased the minimum versions of the following dependencies: + + - service_identity_: 18.1.0 → 23.1.0 + + (:issue:`7347`) + +- Added support for Twisted 26.4.0+. + (:issue:`7347`, :issue:`7505`, :issue:`7520`) + +- Added support for Python 3.14. + (:issue:`6604`, :issue:`7460`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The following classes and functions, intended for internal use by + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have + been made private: + + - ``scrapy.core.downloader.handlers.http11.ScrapyAgent`` + + - ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint`` + + - ``scrapy.core.downloader.handlers.http11.tunnel_request_data()`` + + - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent`` + + (:issue:`7496`, :issue:`7510`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.FormRequest`` is deprecated. You can use the :doc:`form2request + ` library instead, see :ref:`form`. + (:issue:`6438`) + +- ``scrapy.utils.python.MutableChain`` is deprecated. + (:issue:`7504`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider`, deprecated in + 2.13.0, is removed and no longer called. Use :meth:`~scrapy.Spider.start` + instead, or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for ``process_start_requests()`` methods of :ref:`spider middlewares + `, deprecated in 2.13.0, is removed. Use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for synchronous ``process_spider_output()`` methods of spider + middlewares, deprecated in Scrapy 2.13.0, is removed. You should upgrade + the affected middlewares to have asynchronous ``process_spider_output()`` + methods. + (:issue:`7504`) + +- The ``spider`` arguments of the following methods of + :class:`~scrapy.core.scraper.Scraper`, deprecated in Scrapy 2.13.0, are + removed: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`7487`) + +- HTTP/1.0 support code, deprecated in Scrapy 2.13.0, is removed. This + includes: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - The ``scrapy.core.downloader.webclient`` module. + + - The ``DOWNLOADER_HTTPCLIENTFACTORY`` setting. + + (:issue:`7486`) + +- The following functions, deprecated in Scrapy 2.13.0, are removed, you + should import them from :mod:`w3lib.url` directly instead: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + (:issue:`7487`) + +- The following test-related code, deprecated in Scrapy 2.13.0, is removed: + + - the ``scrapy.utils.testproc`` module + + - the ``scrapy.utils.testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + - ``scrapy.utils.test.TestSpider`` + + (:issue:`7487`) + +- ``scrapy.utils.versions.scrapy_components_versions()``, deprecated in + Scrapy 2.13.0, is removed, you can use + :func:`scrapy.utils.versions.get_versions` instead. + (:issue:`7487`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` and + ``scrapy.utils.url.escape_ajax()``, deprecated in Scrapy 2.13.0, are + removed. + (:issue:`7487`) + +- The ``__init__()`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) now needs to support a keyword-only + ``start_queue_cls`` parameter, not supporting it was deprecated in Scrapy + 2.13.0. + (:issue:`7487`) + +- ``scrapy.spiders.init.InitSpider``, deprecated in Scrapy 2.13.0, is + removed. + (:issue:`7487`) + +New features +~~~~~~~~~~~~ + +- New features and improvements for + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`: + + - Support for proxies. + + - Support for the :reqmeta:`download_latency` meta key. + + - Support for :attr:`Response.certificate + `. + + - Default headers set by the ``httpx`` library are no longer added to + requests. + + (:issue:`7441`, :issue:`7524`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + skips HTTPS proxy certificate verification when the + :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting is set to ``False``. + (:issue:`7496`) + +Improvements +~~~~~~~~~~~~ + +- :func:`time.monotonic` is used instead of :func:`time.time` to calculate + elapsed time in various places. + (:issue:`7377`) + +- Improved extraction of the file extension from the URL in + :class:`~scrapy.pipelines.files.FilesPipeline`. + (:issue:`4225`, :issue:`7414`) + +- Other code refactoring and improvements. + (:issue:`7401`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + raises an exception when a request has an ``https://`` destination and an + ``https://`` proxy, which is not supported by this handler. Previously it + tried to connect to the proxy via HTTP in this case. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now + raises an exception for requests with ``http://`` URLs instead of trying to + connect, which is not supported by this handler. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` no longer + adds the ``:status`` pseudo-header to :attr:`Response.headers + `. + (:issue:`7441`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` removing the ```` + tag when adding the ```` tag. + (:issue:`7459`) + +Documentation +~~~~~~~~~~~~~ + +- Documented that + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + doesn't support HTTPS proxies for HTTPS destinations and that + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` doesn't + support proxies at all. + (:issue:`7496`) + +- Added an example of using + :class:`logging.handlers.TimedRotatingFileHandler` to rotate Scrapy logs. + (:issue:`3628`, :issue:`7501`) + +- Added a ``CITATION.cff`` file. + (:issue:`7502`, :issue:`7519`) + +- Mentioned ``DOWNLOADER_CLIENT_TLS_METHOD`` in :ref:`bans`. + (:issue:`5232`, :issue:`7518`) + +- Other documentation improvements and fixes. + (:issue:`7417`, + :issue:`7463`, + :issue:`7472`, + :issue:`7480`, + :issue:`7489`, + :issue:`7503`, + :issue:`7507`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added tests that connect to https://books.toscrape.com/ to test the + behavior with a real website. These tests are marked with the + ``requires_internet`` pytest mark and can be skipped with e.g. + ``-m 'not requires_internet'`` if you cannot or don't want to run them. + (:issue:`7520`) + +- Type hints improvements and fixes. + (:issue:`7492`, :issue:`7532`) + +- CI and test improvements and fixes. + (:issue:`7441`, :issue:`7466`, :issue:`7491`, :issue:`7496`) + +.. _release-2.15.2: + +Scrapy 2.15.2 (2026-04-28) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Fixed links in https://docs.scrapy.org/llms.txt (:issue:`7467`) + +.. _release-2.15.1: + +Scrapy 2.15.1 (2026-04-23) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Sharing of the SSL context between multiple connections, introduced in + Scrapy 2.15.0, is reverted as it caused problems and wasn't actually + needed. + (:issue:`7445`, :issue:`7450`) + +- Fixed :meth:`scrapy.settings.BaseSettings.getwithbase` failing on keys with + dots that aren't import names. It now works the way it worked before Scrapy + 2.15.0, without trying to match class objects and import path. A separate + method, + :func:`~scrapy.settings.BaseSettings.get_component_priority_dict_with_base`, + was added that does that, and it is now used for :ref:`component priority + dictionaries `. + (:issue:`7426`, :issue:`7449`) + +- Documentation rendering improvements. + (:issue:`7452`, :issue:`7454`) + +.. _release-2.15.0: + +Scrapy 2.15.0 (2026-04-09) +-------------------------- + +Highlights: + +- Experimental support for running without a Twisted reactor + +- Experimental ``httpx``-based download handler + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The built-in HTTP :ref:`download handlers ` now + raise Scrapy-specific exceptions instead of implementation-specific ones, + see :ref:`download-handlers-exceptions`. This can affect user code that + handles downloader exceptions, such as ``process_exception()`` methods of + custom :ref:`downloader middlewares `. + (:issue:`7208`) + +- In order to fix a long-standing bug with handling of asynchronous storages, + the following changes were made to media pipeline classes, which can impact + some of the user code that subclasses them or calls their methods directly: + + - overrides of :meth:`scrapy.pipelines.media.MediaPipeline.media_downloaded` + and :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` can now + return coroutines + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`, + :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` and + :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` now + return coroutines + + (:issue:`2183`, :issue:`6369`, :issue:`7182`) + +- ``Request`` and ``Response`` objects: ``__slots__`` and setter changes: + + - :class:`scrapy.http.Request` and :class:`scrapy.http.Response` now + define ``__slots__``. Assigning arbitrary attributes to instances (for + example, ``response.foo = 1``) will raise ``AttributeError``. Store + per-request/response data in the request/response ``meta`` mapping + instead of attaching new attributes to the objects. + + - If you maintain custom ``Request`` or ``Response`` subclasses that + relied on dynamic instance attributes, either add ``'__dict__'`` to + your subclass ``__slots__`` to allow dynamic attributes, or migrate + per-instance state to ``meta`` or explicit documented attributes. + + - The setters for ``headers``, ``flags`` and ``cookies`` no longer coerce + falsy values into ``None``. For example, ``request.headers = {}`` now + stores an empty :class:`scrapy.http.headers.Headers` instance (not + ``None``), and ``request.flags = []`` remains an empty list instead of + being set to ``None``. Update code that relied on ``is None`` checks or + the previous coercion behaviour. + + (:issue:`7036`, :issue:`7367`, :issue:`7374`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The context factory class set as the value of the + ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is now required to support the + ``method`` argument of ``__init__()``, recommended since Scrapy 1.2.0. + (:issue:`7353`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.mail.MailSender`` is deprecated. Please use :mod:`smtplib`, + :mod:`twisted.mail.smtp` or other 3rd party email libraries. + (:issue:`7249`, :issue:`7263`) + +- The ``scrapy.extensions.statsmailer.StatsMailer`` extension is deprecated. + You can instead implement your own notifications by handling the + :signal:`spider_closed` signal. + (:issue:`7249`, :issue:`7263`) + +- The ``MEMUSAGE_NOTIFY_MAIL`` setting is deprecated. You can instead + implement your own notifications by handling the + :signal:`memusage_warning_reached` and :signal:`spider_closed` signals. + (:issue:`7249`, :issue:`7263`) + +- The ``DNS_RESOLVER`` setting was renamed to :setting:`TWISTED_DNS_RESOLVER` + and the old name is deprecated. + (:issue:`7350`, :issue:`7361`) + +- The ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is deprecated. If you were + using it to switch to + ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory``, please + use the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting instead. If you + cannot use the default context factory for some other reason, please + subclass the :ref:`download handler ` instead. + (:issue:`7352`, :issue:`7379`) + +- ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory`` is + deprecated. You can set the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` + setting to ``True`` instead. + (:issue:`7379`) + +- The following implementation details of the context factory handling code + are deprecated: + + - ``scrapy.core.downloader.contextfactory.AcceptableProtocolsContextFactory`` + + - ``scrapy.core.downloader.contextfactory.load_context_factory_from_settings()`` + + - ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory`` + + - ``scrapy.core.downloader.tls.ScrapyClientTLSOptions`` + + (:issue:`7353`, :issue:`7391`) + +- Passing :class:`str` instead of :class:`bytes` to + :class:`scrapy.utils.sitemap.Sitemap` and + :func:`scrapy.utils.sitemap.sitemap_urls_from_robots` is deprecated. + (:issue:`7007`) + +- ``scrapy.utils.misc.walk_modules()`` is deprecated. You can use + :func:`scrapy.utils.misc.walk_modules_iter` instead. + (:issue:`7388`) + +- ``scrapy.shell.Shell.inthread`` is deprecated. You can use + :attr:`scrapy.shell.Shell.fetch_available` instead to check if + :func:`~scrapy.shell.Shell.fetch` can be used. + (:issue:`7395`) + +- ``scrapy.commands.ScrapyCommand.set_crawler()`` is deprecated. + (:issue:`7276`) + +New features +~~~~~~~~~~~~ + +- Added an *experimental* mode for running Scrapy without installing a + Twisted reactor: set :setting:`TWISTED_REACTOR_ENABLED` to ``False`` to + enable it. This mode has limitations, refer to :ref:`its documentation + ` for details. As long as it's experimental, its + behavior and related features and APIs may change in future Scrapy releases + in a breaking way. + (:issue:`6219`, + :issue:`7185`, + :issue:`7186`, + :issue:`7187`, + :issue:`7188`, + :issue:`7190`, + :issue:`7197`, + :issue:`7199`, + :issue:`7209`, + :issue:`7228`, + :issue:`7355`, + :issue:`7366`, + :issue:`7385`, + :issue:`7395`) + +- Added the :func:`scrapy.utils.reactorless.is_reactorless` function that + checks if there is a running asyncio event loop but no Twisted reactor. + (:issue:`7185`, :issue:`7199`) + +- Changed :func:`scrapy.utils.asyncio.is_asyncio_available` to return + ``True`` if there is a running asyncio loop, even if no Twisted reactor is + installed. + (:issue:`7185`, :issue:`7199`) + +- Added an *experimental* download handler that uses the httpx_ library and + doesn't require a Twisted reactor: + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. As + long as it's experimental, its behavior may change in future Scrapy + releases in a breaking way. + (:issue:`6805`, :issue:`7239`, :issue:`7368`, :issue:`7384`) + + .. _httpx: https://www.python-httpx.org/ + +- Added the :setting:`DOWNLOAD_BIND_ADDRESS` setting as a global counterpart + to the per-request :reqmeta:`bindaddress` meta key. + (:issue:`7266`, :issue:`7283`) + +- Added the :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting that can be set + to ``True`` to make Scrapy abort HTTPS requests when the server certificate + is invalid or doesn't match the domain. + (:issue:`7379`) + +- The built-in HTTP :ref:`download handlers ` now + raise Scrapy-specific exceptions instead of implementation-specific ones, + to allow unified handling of similar problems caused by different + implementations. The default value of the :setting:`RETRY_EXCEPTIONS` + setting was updated replacing Twisted-specific exceptions with these new + ones. The exceptions: + + - :exc:`~scrapy.exceptions.CannotResolveHostError` + + - :exc:`~scrapy.exceptions.DownloadCancelledError` + + - :exc:`~scrapy.exceptions.DownloadConnectionRefusedError` + + - :exc:`~scrapy.exceptions.DownloadFailedError` + + - :exc:`~scrapy.exceptions.DownloadTimeoutError` + + - :exc:`~scrapy.exceptions.ResponseDataLossError` + + - :exc:`~scrapy.exceptions.UnsupportedURLSchemeError` + + (:issue:`7208`) + +- Added the :signal:`memusage_warning_reached` signal emitted by the + :class:`~scrapy.extensions.memusage.MemoryUsage` extension when the memory + usage reaches :setting:`MEMUSAGE_WARNING_MB`. + (:issue:`7249`, :issue:`7263`) + +- Added + :meth:`Headers.to_tuple_list() ` + that returns headers as a list of ``(key, value)`` tuples. + (:issue:`7239`) + +- :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` now uses the + download handler configured for the ``"https"`` scheme to make requests + instead of always using + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + (:issue:`7369`, :issue:`7370`) + +- Added :func:`scrapy.utils.misc.walk_modules_iter` as a replacement for + ``scrapy.utils.misc.walk_modules()`` that returns an iterable instead of a + list. + (:issue:`7388`) + +Improvements +~~~~~~~~~~~~ + +- :func:`asyncio.to_thread` is now used instead of + :func:`twisted.internet.threads.deferToThread` in the built-in feed + storages, media pipeline storages and the + :func:`scrapy.utils.decorators.inthread` decorator when available. + (:issue:`7183`, :issue:`7184`, :issue:`7349`) + +- Improved memory footprint of :class:`~scrapy.Request` and + :class:`~scrapy.http.Response` objects by adding ``__slots__`` and omitting + empty lists and dicts in some internal attributes. + (:issue:`7036`, :issue:`7367`, :issue:`7374`) + +- :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory` + no longer mutates the SSL context, to avoid the behavior that was + deprecated in pyOpenSSL 25.1.0. + (:issue:`6859`, :issue:`7353`) + +- Improved memory usage of :class:`~scrapy.spiders.sitemap.SitemapSpider` and + :class:`scrapy.utils.sitemap.Sitemap`. + (:issue:`3529`, :issue:`7007`) + +- Improved the scheduling behavior of + :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` when crawling + multiple domains. + (:issue:`7293`, :issue:`7351`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now handle + TLS verbose logging (see :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING`) + directly instead of relying on + :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory`. + (:issue:`7387`) + +- The server certificate verification code now correctly handles certificates + with IP addresses in ``subjectAltName``. + (:issue:`7353`) + +- Improved reliability of :func:`scrapy.utils.trackref.get_oldest`. + (:issue:`1758`, :issue:`7375`) + +- Other code refactoring and improvements. + (:issue:`7210`, :issue:`7238`, :issue:`7376`, :issue:`7386`, :issue:`7395`, + :issue:`7405`, :issue:`7410`) + +Bug fixes +~~~~~~~~~ + +- :ref:`Media pipelines ` should now wait for uploads + to asynchronous storages (e.g. + :class:`~scrapy.pipelines.files.S3FilesStore`) to complete. + (:issue:`2183`, :issue:`6369`, :issue:`7182`) + +- Fixed merging ``*_BASE`` settings (e.g. merging + :setting:`DOWNLOADER_MIDDLEWARES` with + :setting:`DOWNLOADER_MIDDLEWARES_BASE`) when a component is referred to by + a class object in one setting and by a string import path in the other one. + (:issue:`6912`, :issue:`6993`) + +- ``scrapy runspider`` and ``scrapy crawl`` now set the exit code to 1 if an + exception happened early (this was broken since Scrapy 2.13.0). + (:issue:`6820`, :issue:`7255`) + +- Fixed repeated warnings about data loss (see + :setting:`DOWNLOAD_FAIL_ON_DATALOSS`) not being suppressed in + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + (:issue:`7222`) + +- Improved FTP connection management in + :class:`scrapy.pipelines.files.FTPFilesStore`. + (:issue:`7256`) + +- Fixed the ``spider`` variable in the :ref:`shell `, which + wasn't available since Scrapy 2.13.0. + (:issue:`7395`) + +Documentation +~~~~~~~~~~~~~ + +- The ``llms.txt`` and ``llms-full.txt`` files and Markdown versions of pages + are now generated when the HTML documentation is built. + (:issue:`7380`) + +- Added a "Copy as Markdown" button to the HTML documentation. + (:issue:`7380`) + +- Added :ref:`docs for using Pydantic models as items `. + (:issue:`6955`, :issue:`6966`) + +- Documented :ref:`job directory contents `. + (:issue:`4842`, :issue:`5260`) + +- Improved docs for :attr:`~scrapy.Request.dont_filter`. + (:issue:`6398`, :issue:`7245`) + +- Clarified that settings related to :setting:`TWISTED_DNS_RESOLVER` are only + taken into account if the selected resolver supports them. + (:issue:`7385`) + +- Other documentation improvements and fixes. + (:issue:`7248`, :issue:`7274`, :issue:`7406`, :issue:`7408`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added the ``no-reactor`` test environment that doesn't install a Twisted + reactor and uses ``pytest-asyncio`` instead of ``pytest-twisted`` to run + asynchronous test functions. + (:issue:`6952`, :issue:`7189`, :issue:`7233`, :issue:`7234`, :issue:`7254`, + :issue:`7259`) + +- Fixed running tests with ``pytest-xdist``. + (:issue:`7216`, :issue:`7257`) + +- Type hints improvements and fixes. + (:issue:`7300`, :issue:`7331`) + +- CI and test improvements and fixes. + (:issue:`7060`, + :issue:`7223`, + :issue:`7232`, + :issue:`7241`, + :issue:`7250`, + :issue:`7256`, + :issue:`7276`, + :issue:`7277`, + :issue:`7279`, + :issue:`7329`, + :issue:`7363`, + :issue:`7381`, + :issue:`7402`) + +.. _release-2.14.2: + +Scrapy 2.14.2 (2026-03-12) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Values from the ``Referrer-Policy`` header of HTTP responses are no longer + executed as Python callables. See the `cwxj-rr6w-m6w7`_ security advisory + for details. + + .. _cwxj-rr6w-m6w7: https://github.com/scrapy/scrapy/security/advisories/GHSA-cwxj-rr6w-m6w7 + +- In line with the `standard + `__, 301 redirects of + ``POST`` requests are converted into ``GET`` requests. + + Converting to a ``GET`` request implies not only a method change, but also + omitting the body and ``Content-*`` headers in the redirect request. On + cross-origin redirects (for example, cross-domain redirects), this is + effectively a security bug fix for scenarios where the body contains + secrets. + +Deprecations +~~~~~~~~~~~~ + +- Passing a response URL string as the first positional argument to + :meth:`scrapy.spidermiddlewares.referer.RefererMiddleware.policy` is + deprecated. Pass a :class:`~scrapy.http.Response` instead. + + The parameter has also been renamed to ``response`` to reflect this change. + The old parameter name (``resp_or_url``) is deprecated. + +New features +~~~~~~~~~~~~ + +- Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing + supported referrer policies. + +Bug fixes +~~~~~~~~~ + +- Made additional redirect scenarios convert to ``GET`` in line with the + `standard `__: + + - Only ``POST`` 302 redirects are converted into ``GET`` requests; other + methods are preserved. + + - ``HEAD`` 303 redirects are not converted into ``GET`` requests. + + - ``GET`` 303 redirects do not have their body or standard ``Content-*`` + headers removed. + +- Redirects where the original request body is dropped now also have their + ``Content-Encoding``, ``Content-Language`` and ``Content-Location`` headers + removed, in addition to the ``Content-Type`` and ``Content-Length`` headers + that were already being removed. + +- Redirects now preserve the source URL fragment if the redirect URL does not + include one. This is useful when using browser-based download handlers, + such as `scrapy-playwright`_ or `scrapy-zyte-api`_, while letting Scrapy + handle redirects. + + .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright + .. _scrapy-zyte-api: https://scrapy-zyte-api.readthedocs.io/en/latest/ + +- The ``Referer`` header is now removed on redirect if + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` is disabled. + +- The handling of the ``Referer`` header on redirects now takes into account + the ``Referer-Policy`` header of the response that triggers the redirect. + +.. _release-2.14.1: + +Scrapy 2.14.1 (2026-01-12) +-------------------------- + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.utils.defer.maybeDeferred_coro()`` is deprecated. (:issue:`7212`) + +Bug fixes +~~~~~~~~~ + +- Fixed custom stats collectors that require a ``spider`` argument in their + ``open_spider()`` and ``close_spider()`` methods not receiving the + argument when called by the engine. + + Note, however, that the ``spider`` argument is now deprecated and will stop + being passed in a future version of Scrapy. + + (:issue:`7213`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Replaced deprecated ``codecov/test-results-action@v1`` GitHub Action with + ``codecov/codecov-action@v5``. + (:issue:`7180`, :issue:`7215`) + +.. _release-2.14.0: + +Scrapy 2.14.0 (2026-01-05) +-------------------------- + +Highlights: + +- More coroutine-based replacements for Deferred-based APIs + +- The default priority queue is now ``DownloaderAwarePriorityQueue`` + +- Dropped support for Python 3.9 and PyPy 3.10 + +- Improved and documented the API for custom download handlers + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.9. + (:issue:`7121`) + +- Dropped support for PyPy 3.10. + (:issue:`7050`) + +- Increased the minimum versions of the following dependencies: + + - lxml_: 4.6.0 → 4.6.4 + + - Pillow_ (optional dependency): 8.0.0 → 8.3.2 + + - botocore_ (optional dependency): 1.4.87 → 1.13.45 + +- Restored support for ``brotlicffi`` dropped in Scrapy 2.13.4. Its minimum + supported version is now ``1.2.0.0``. + (:issue:`7160`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- If you set the :setting:`TWISTED_REACTOR` setting to a :ref:`non-asyncio + value ` at the :ref:`spider level `, you + may now need to set the :setting:`FORCE_CRAWLER_PROCESS` setting to + ``True`` when running Scrapy via :ref:`its command-line tool + ` to avoid a reactor mismatch exception. + (:issue:`6845`) + +- The ``log_count/*`` stats no longer count some of the early messages that + they counted before. While the earliest log messages, emitted before the + counter is initialized, were never counted, the counter initialization now + happens later than in previous Scrapy versions. You may need to adjust + expected values if you retrieve and compare values of these stats in your + code. + (:issue:`7046`) + +- The classes listed below are now :term:`abstract base classes `. They cannot be instantiated directly and their subclasses + need to override the abstract methods listed below to be able to be + instantiated. If you previously instantiated these classes directly, you + will now need to subclass them and provide trivial (e.g. empty) + implementations for the abstract methods. + + - :class:`scrapy.commands.ScrapyCommand` + + - :meth:`~scrapy.commands.ScrapyCommand.run` + + - :meth:`~scrapy.commands.ScrapyCommand.short_desc` + + - :class:`scrapy.exporters.BaseItemExporter` + + - :meth:`~scrapy.exporters.BaseItemExporter.export_item` + + - :class:`scrapy.extensions.feedexport.BlockingFeedStorage` + + - :meth:`~scrapy.extensions.feedexport.BlockingFeedStorage._store_in_thread` + + - :class:`scrapy.middleware.MiddlewareManager` + + - :meth:`~scrapy.middleware.MiddlewareManager._get_mwlist_from_settings` + + - :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` + + - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer` + + (:issue:`6930`) + +- Scrapy no longer passes a ``spider`` argument to any methods of the + :setting:`stats collector `. It wasn't passed in many of the + calls even in older Scrapy versions, so we don't expect existing custom + stats collector implementations to require a ``spider`` argument. If your + implementation needs a :class:`~scrapy.Spider` instance, you can get it + from the :class:`~scrapy.crawler.Crawler` instance passed to the + constructor. + (:issue:`7011`) + +- :class:`scrapy.middleware.MiddlewareManager` no longer includes code for + handling ``open_spider()`` and ``close_spider()`` component methods. As + this code was only used for pipelines it was moved into + :class:`scrapy.pipelines.ItemPipelineManager`. This change should only + affect custom subclasses of :class:`~scrapy.middleware.MiddlewareManager`. + The following code was moved: + + - ``scrapy.middleware.MiddlewareManager.open_spider()`` + + - ``scrapy.middleware.MiddlewareManager.close_spider()`` + + - Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that + processes ``open_spider()`` and ``close_spider()`` component methods. + + (:issue:`7006`) + +- :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request` + now returns a coroutine, previously it returned a + :class:`~twisted.internet.defer.Deferred` object or ``None``. The + ``robot_parser()`` method was also changed to return a coroutine. This + change only impacts code that subclasses + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` or + calls its methods directly. + (:issue:`6802`) + +- The built-in :ref:`download handlers ` have been + refactored, changing the signatures of their methods. This change should + only affect user code that subclasses any of these handlers or calls their + methods directly. + (:issue:`6778`, :issue:`7164`) + +- :meth:`scrapy.pipelines.media.MediaPipeline.process_item` now returns a + coroutine, previously it returned a + :class:`~twisted.internet.defer.Deferred` object. This + change only impacts code that calls this method directly. + (:issue:`7177`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``from_settings()`` method of the following components, deprecated in + Scrapy 2.12.0, is removed. You should use ``from_crawler()`` instead. + + - :class:`scrapy.dupefilters.RFPDupeFilter` + - :class:`scrapy.mail.MailSender` + - :class:`scrapy.middleware.MiddlewareManager` + - :class:`scrapy.core.downloader.contextfactory.ScrapyClientContextFactory` + - :class:`scrapy.pipelines.files.FilesPipeline` + - :class:`scrapy.pipelines.images.ImagesPipeline` + + (:issue:`7126`) + +- Scrapy no longer calls ``from_settings()`` methods of 3rd-party + :ref:`components `, deprecated in Scrapy 2.12.0. You + should define a ``from_crawler()`` method instead. + (:issue:`7126`) + +- The initialization flow of :class:`scrapy.pipelines.media.MediaPipeline` + and its subclasses was simplified, it now mandates ``from_crawler()`` + methods and ``crawler`` arguments of ``__init__()`` methods. Not using + these was deprecated in Scrapy 2.12.0. + (:issue:`7126`) + +- The ``REQUEST_FINGERPRINTER_IMPLEMENTATION`` setting, deprecated in Scrapy + 2.12.0, is removed. + (:issue:`7126`) + +- The ``scrapy.utils.misc.create_instance()`` function, deprecated in Scrapy + 2.12.0, is removed. Use :func:`scrapy.utils.misc.build_from_crawler` + instead. + (:issue:`7126`) + +- The ``scrapy.core.downloader.Downloader._get_slot_key()`` function, + deprecated in Scrapy 2.12.0, is removed. Use + :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. + (:issue:`7126`) + +- The ``scrapy.twisted_version`` attribute, deprecated in Scrapy 2.12.0, is + removed. You should instead use the :attr:`twisted.version` attribute + directly. + (:issue:`7126`) + +- The following utility functions, deprecated in Scrapy 2.12.0, are removed: + + - ``scrapy.utils.defer.process_chain_both()`` + - ``scrapy.utils.python.equal_attributes()`` + - ``scrapy.utils.python.flatten()`` + - ``scrapy.utils.python.iflatten()`` + - ``scrapy.utils.request.request_authenticate()`` + - ``scrapy.utils.test.assert_samelines()`` + + (:issue:`7126`) + +- ``scrapy.utils.serialize.ScrapyJSONDecoder``, deprecated in Scrapy 2.12.0, + is removed. + (:issue:`7126`) + +- The ``scrapy.extensions.feedexport.build_storage()`` function, deprecated + in Scrapy 2.12.0, is removed, you can instead call the builder callable + directly. + (:issue:`7126`) + +- ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware``, deprecated in + Scrapy 2.11.2, is removed. + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` should be + used instead. + (:issue:`6926`) + +Deprecations +~~~~~~~~~~~~ + +- The following methods that return a + :class:`~twisted.internet.defer.Deferred` are deprecated in favor of their + coroutine-based replacements: + + - :class:`scrapy.core.downloader.handlers.DownloadHandlers` + + - ``download_request()`` (use + :meth:`~scrapy.core.downloader.handlers.DownloadHandlers.download_request_async`) + + - :class:`scrapy.core.downloader.middleware.DownloaderMiddlewareManager` + + - ``download()`` (use + :meth:`~scrapy.core.downloader.middleware.DownloaderMiddlewareManager.download_async`) + + - :class:`scrapy.core.engine.ExecutionEngine` + + - ``start()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.start_async`) + + - ``stop()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.stop_async`) + + - ``close()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.close_async`) + + - ``open_spider()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.close_spider_async`) + + - ``download()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.download_async`) + + - :class:`scrapy.core.scraper.Scraper` + + - ``open_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.open_spider_async`) + + - ``call_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.call_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.close_spider_async`) + + - ``handle_spider_output()`` (use + :meth:`~scrapy.core.scraper.Scraper.handle_spider_output_async`) + + - ``start_itemproc()`` (use + :meth:`~scrapy.core.scraper.Scraper.start_itemproc_async`) + + - :class:`scrapy.core.spidermw.SpiderMiddlewareManager` + + - ``scrape_response()`` (use + :meth:`~scrapy.core.spidermw.SpiderMiddlewareManager.scrape_response_async`) + + - :class:`scrapy.crawler.Crawler` + + - ``stop()`` (use :meth:`~scrapy.crawler.Crawler.stop_async`) + + - :class:`scrapy.pipelines.ItemPipelineManager` + + - ``process_item()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.process_item_async`) + + - ``open_spider()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.open_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.close_spider_async`) + + - :class:`scrapy.signalmanager.SignalManager` + + - ``send_catch_log_deferred()`` (use + :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_async`) + + - ``scrapy.utils.signal.send_catch_log_deferred()`` (use + :func:`scrapy.utils.signal.send_catch_log_async`) + + (:issue:`6791`, :issue:`6842`, :issue:`6979`, :issue:`6997`, :issue:`6999`, + :issue:`7005`, :issue:`7043`, :issue:`7069`, :issue:`7161`, :issue:`7164`) + +- The following spider attributes are deprecated in favor of settings: + + - ``download_maxsize`` (use :setting:`DOWNLOAD_MAXSIZE`) + + - ``download_timeout`` (use :setting:`DOWNLOAD_TIMEOUT`) + + - ``download_warnsize`` (use :setting:`DOWNLOAD_WARNSIZE`) + + - ``max_concurrent_requests`` (use + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) + + - ``user_agent`` (use :setting:`USER_AGENT`) + + (:issue:`6988`, :issue:`6994`, :issue:`7038`, :issue:`7039`, :issue:`7117`, + :issue:`7176`) + +- Returning a :class:`~twisted.internet.defer.Deferred` from the following + user-defined functions is deprecated in favor of defining them as coroutine + functions: + + - spider callbacks and errbacks (which was never officially supported and + may work incorrectly) + + - the ``process_request()``, ``process_response()`` and + ``process_exception()`` methods of custom downloader middlewares + + - the ``process_item()``, ``open_spider()`` and ``close_spider()`` methods + of custom pipelines + + - signal handlers + + - the ``download_request()`` and ``close()`` methods of custom download + handlers + + (:issue:`6718`, :issue:`6778`, :issue:`7069`, :issue:`7147`, :issue:`7148`, + :issue:`7149`, :issue:`7150`, :issue:`7151`, :issue:`7161`, :issue:`7164`, + :issue:`7179`) + +- Passing a ``spider`` argument to the following methods is deprecated: + + - :meth:`scrapy.core.spidermw.SpiderMiddlewareManager.process_start` + + - :meth:`scrapy.core.downloader.Downloader.fetch` + + - :meth:`scrapy.core.downloader.Downloader._get_slot` + + - :meth:`scrapy.core.downloader.handlers.DownloadHandlers.download_request` + + - all public methods of :class:`scrapy.statscollectors.StatsCollector` + + - :meth:`scrapy.spidermiddlewares.base.BaseSpiderMiddleware.process_spider_output` + + - :meth:`scrapy.spidermiddlewares.base.BaseSpiderMiddleware.process_spider_output_async` + + - all ``process_*()`` methods of built-in downloader middlewares + + - all ``process_*()`` methods of built-in spider middlewares + + - :meth:`scrapy.pipelines.media.MediaPipeline.open_spider` + + - :meth:`scrapy.pipelines.media.MediaPipeline.process_item` + + (:issue:`6750`, :issue:`6927`, :issue:`6984`, :issue:`7006`, :issue:`7011`, + :issue:`7033`, :issue:`7037`, :issue:`7045`, :issue:`7178`) + +- Instantiating subclasses of :class:`scrapy.middleware.MiddlewareManager` + without a :class:`~scrapy.crawler.Crawler` instance is deprecated. + (:issue:`6984`) + +- For the following user-defined functions and methods requiring a ``spider`` + argument is deprecated, if you need a :class:`~scrapy.Spider` instance + inside them you should get it from the :class:`~scrapy.crawler.Crawler` + instance (you may need to refactor your code to save that instance in e.g. + the ``from_crawler()`` method): + + - the ``process_request()``, ``process_response()`` and + ``process_exception()`` methods of custom downloader middlewares + + - the ``process_spider_input()``, ``process_spider_output()``, + ``process_spider_output_async()`` and ``process_spider_exception()`` + methods of custom spider middlewares + + - the ``process_item()`` method of custom pipelines + + - the ``fetch()`` method of a custom :setting:`DOWNLOADER` + + (:issue:`6927`, :issue:`6984`, :issue:`7006`, :issue:`7037`) + +- The following things in custom download handlers are deprecated: + + - not having a ``lazy`` attribute (you should define it as ``True`` if you + want to keep the current behavior) + + - returning a :class:`~twisted.internet.defer.Deferred` from the + ``download_request()`` method (you should refactor it to return a + coroutine; you also need to remove the ``spider`` argument when doing + this) + + - not having a ``close()`` method, having a synchronous one or one that + returns a :class:`~twisted.internet.defer.Deferred` (you should refactor + it to return a coroutine or add an empty one if you don't have it) + + (:issue:`6778`, :issue:`7164`) + +- Custom implementations of :setting:`ITEM_PROCESSOR` should now define + ``process_item_async()``, ``open_spider_async()`` and + ``close_spider_async()`` methods instead of, or in addition to, + ``process_item()``, ``open_spider()`` and ``close_spider()``. + (:issue:`7005`, :issue:`7043`) + +- The ``CONCURRENT_REQUESTS_PER_IP`` setting is deprecated, use + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` instead. + (:issue:`6917`, :issue:`6921`) + +- The ``scrapy.core.downloader.handlers.http`` module is deprecated. You + should import + :class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + directly instead of importing the + ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler`` alias. + (:issue:`7079`) + +- The ``scrapy.utils.decorators.defers()`` decorator is deprecated, you can + use :func:`twisted.internet.defer.maybeDeferred` directly or reimplement + this decorator in your code. + (:issue:`7164`) + +- ``scrapy.spiders.CrawlSpider._parse_response()`` is deprecated, use + :meth:`scrapy.spiders.CrawlSpider.parse_with_rules` instead. + (:issue:`4463`, :issue:`6804`) + +- The functions that add a delay to a Deferred are deprecated, their + underlying Twisted functions can be used instead, either directly if a + delay isn't needed, or with some explicit way to add a delay if it's + needed: + + - ``scrapy.utils.defer.mustbe_deferred()`` (you can use + :func:`twisted.internet.defer.maybeDeferred`) + + - ``scrapy.utils.defer.defer_succeed()`` (you can use + :func:`twisted.internet.defer.succeed`) + + - ``scrapy.utils.defer.defer_fail()`` (you can use + :func:`twisted.internet.defer.fail`) + + - ``scrapy.utils.defer.defer_result()`` (you can use + :func:`twisted.internet.defer.succeed` and + :func:`twisted.internet.defer.fail`) + + (:issue:`6937`) + +New features +~~~~~~~~~~~~ + +- Added :class:`scrapy.crawler.AsyncCrawlerProcess` and + :class:`scrapy.crawler.AsyncCrawlerRunner` as counterparts to + :class:`~scrapy.crawler.CrawlerProcess` and + :class:`~scrapy.crawler.CrawlerRunner` that offer coroutine-based APIs. + (:issue:`6789`, :issue:`6790`, :issue:`6796`, :issue:`6817`, :issue:`6845`, + :issue:`7034`) + +- Added coroutine counterparts to some of the Deferred-based APIs: + + - :class:`scrapy.core.downloader.handlers.DownloadHandlers` + + - :meth:`~scrapy.core.downloader.handlers.DownloadHandlers.download_request_async` + (to ``download_request()``) + + - :class:`scrapy.core.downloader.middleware.DownloaderMiddlewareManager` + + - :meth:`~scrapy.core.downloader.middleware.DownloaderMiddlewareManager.download_async` + (to ``download()``) + + - :class:`scrapy.core.engine.ExecutionEngine` + + - :meth:`~scrapy.core.engine.ExecutionEngine.start_async` (to + ``start()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.stop_async` (to + ``stop()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.close_async` (to + ``close()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.close_spider_async` (to + ``close_spider()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.download_async` (to + ``download()``) + + - :class:`scrapy.core.scraper.Scraper` + + - :meth:`~scrapy.core.scraper.Scraper.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.core.scraper.Scraper.close_spider_async` (to + ``close_spider()``) + + - :meth:`~scrapy.core.scraper.Scraper.start_itemproc_async` (to + ``start_itemproc()``) + + - :class:`scrapy.crawler.Crawler` + + - :meth:`~scrapy.crawler.Crawler.crawl_async` (to ``crawl()``) + + - :meth:`~scrapy.crawler.Crawler.stop_async` (to ``stop()``) + + - :class:`scrapy.pipelines.ItemPipelineManager` + + - :meth:`~scrapy.pipelines.ItemPipelineManager.process_item_async` (to + ``process_item()``) + + - :meth:`~scrapy.pipelines.ItemPipelineManager.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.pipelines.ItemPipelineManager.close_spider_async` (to + ``close_spider()``) + + - :class:`scrapy.signalmanager.SignalManager` + + - :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_async` (to + ``send_catch_log_deferred()``) + + (:issue:`6781`, :issue:`6791`, :issue:`6792`, :issue:`6795`, :issue:`6801`, + :issue:`6817`, :issue:`6842`, :issue:`6997`, :issue:`7005`, :issue:`7043`, + :issue:`7069`,:issue:`7164`, :issue:`7202`) + +- The default value of the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is now + ``'scrapy.pqueues.DownloaderAwarePriorityQueue'``. + (:issue:`6924`, :issue:`6940`) + +- Added :class:`scrapy.extensions.logcount.LogCount`, an enabled-by-default + extension that is responsible for the ``log_count/*`` stats. Previously, + this code was in :class:`scrapy.crawler.Crawler` and couldn't be disabled. + (:issue:`7046`) + +- Added :meth:`scrapy.spiders.CrawlSpider.parse_with_rules` as a public + replacement for ``_parse_response()``. + (:issue:`4463`, :issue:`6804`) + +- Added :func:`scrapy.utils.asyncio.is_asyncio_available` as an alternative + to :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` with a + future-proof name and semantics. + (:issue:`6827`) + +- The API for :ref:`download handlers `, previously + undocumented, has been modernized and documented. An optional base class, + :class:`scrapy.core.downloader.handlers.base.BaseDownloadHandler`, has been + added to simplify writing custom download handlers that conform to the + current API. + (:issue:`4944`, :issue:`6778`, :issue:`7164`) + +- Added :func:`scrapy.utils.defer.ensure_awaitable`, which can be helpful to + call user-defined functions that can return coroutines, Deferreds or + values directly. + (:issue:`7005`) + +- The ``requests.seen`` file, written by + :class:`~scrapy.dupefilters.RFPDupeFilter` when :ref:`job persistence + ` is enabled, now uses line buffering to reduce data loss in + spider crashes. + (:issue:`6019`, :issue:`7094`) + +- Images downloaded by :class:`~scrapy.pipelines.images.ImagesPipeline` are + now automatically transposed based on EXIF data. + (:issue:`6525`, :issue:`6975`) + +Improvements +~~~~~~~~~~~~ + +- Refactored internal functions to use coroutines instead of Deferreds. + (:issue:`6795`, :issue:`6852`, :issue:`6855`, :issue:`6858`, :issue:`7159`) + +- Commands that don't need a :class:`~scrapy.crawler.CrawlerProcess` instance + no longer create it. + (:issue:`6824`) + +- Improved :command:`shell` help formatting when using IPython 9+. + (:issue:`6915`, :issue:`6980`) + +Bug fixes +~~~~~~~~~ + +- Setting :setting:`FILES_STORE` or :setting:`IMAGES_STORE` to ``None`` now + correctly disables the respective pipeline. + (:issue:`6964`, :issue:`6969`) + +- :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` now + uses the URL set in the ```` tag as the base URL when redirecting to + a relative URL. + (:issue:`7042`, :issue:`7047`) + +- Passing ``None`` as a value of the :reqmeta:`download_slot` request meta + key is now handled in the same way as not setting this meta key at all. + (:issue:`7172`) + +- Fixed parsing of the first line of ``robots.txt`` files that have a BOM. + (:issue:`6195`, :issue:`7095`) + +Documentation +~~~~~~~~~~~~~ + +- Added :ref:`documentation ` about download + handlers, their API and built-in handlers. + (:issue:`4944`, :issue:`7164`) + +- Added a section about the `scrapy-spider-metadata`_ library to the + :ref:`spider argument docs `. + (:issue:`6676`, :issue:`6957`, :issue:`7116`) + + .. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/ + +- Improved :ref:`the docs ` about coroutine-based + and Deferred-based APIs. + (:issue:`6800`, :issue:`7146`) + +- Other documentation improvements and fixes. + (:issue:`7058`, :issue:`7076`, :issue:`7109`, :issue:`7195`, :issue:`7198`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Switched from ``twisted.trial`` to ``pytest-twisted`` and replaced + remaining ``unittest`` and ``twisted.trial`` features with ``pytest`` ones. + (:issue:`6658`, :issue:`6873`, :issue:`6884`, :issue:`6938`) + +- Enabled fancy ``pytest`` asserts. + (:issue:`6888`) + +- Added `Sphinx Lint`_ to the ``pre-commit`` configuration. + (:issue:`6920`) + + .. _Sphinx Lint: https://github.com/sphinx-contrib/sphinx-lint + +- CI and test improvements and fixes. + (:issue:`6649`, + :issue:`6769`, + :issue:`6821`, + :issue:`6835`, + :issue:`6836`, + :issue:`6846`, + :issue:`6883`, + :issue:`6885`, + :issue:`6889`, + :issue:`6905`, + :issue:`6928`, + :issue:`6933`, + :issue:`6941`, + :issue:`6942`, + :issue:`6945`, + :issue:`6947`, + :issue:`6960`, + :issue:`6968`, + :issue:`6972`, + :issue:`6974`, + :issue:`6996`, + :issue:`7003`, + :issue:`7012`, + :issue:`7013`, + :issue:`7050`, + :issue:`7059`, + :issue:`7070`, + :issue:`7073`, + :issue:`7118`, + :issue:`7127`, + :issue:`7141`, + :issue:`7143`, + :issue:`7145`, + :issue:`7173`) + +- Code cleanups. + (:issue:`6803`, + :issue:`6838`, + :issue:`6849`, + :issue:`6875`, + :issue:`6876`, + :issue:`6892`, + :issue:`6930`, + :issue:`6949`, + :issue:`6970`, + :issue:`6977`, + :issue:`6986`, + :issue:`7008`, + :issue:`7177`) + +.. _release-2.13.4: + +Scrapy 2.13.4 (2025-11-17) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Improved protection against decompression bombs in + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + for responses compressed using the ``br`` and ``deflate`` methods: if a + single compressed chunk would be larger than the response size limit (see + :setting:`DOWNLOAD_MAXSIZE`) when decompressed, decompression is no longer + carried out. This is especially important for the ``br`` (Brotli) method + that can provide a very high compression ratio. Please, see the + `CVE-2025-6176`_ and `GHSA-2qfp-q593-8484`_ security advisories for more + information. + (:issue:`7134`) + + .. _CVE-2025-6176: https://nvd.nist.gov/vuln/detail/CVE-2025-6176 + .. _GHSA-2qfp-q593-8484: https://github.com/advisories/GHSA-2qfp-q593-8484 + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The minimum supported version of the optional ``brotli`` package is now + ``1.2.0``. + (:issue:`7134`) + +- The ``brotlicffi`` and ``brotlipy`` packages can no longer be used to + decompress Brotli-compressed responses. Please install the ``brotli`` + package instead. + (:issue:`7134`) + +Other changes +~~~~~~~~~~~~~ + +- Restricted the maximum supported Twisted version to ``25.5.0``, as Scrapy + currently uses some private APIs changed in later Twisted versions. + (:issue:`7142`) + +- Stopped setting the ``COVERAGE_CORE`` environment variable in tests, it + didn't have an effect but caused the ``coverage`` module to produce a + warning or an error. + (:issue:`7137`) + +- Removed the documentation build dependency on the deprecated + ``sphinx-hoverxref`` module. + (:issue:`6786`, :issue:`6922`) + +.. _release-2.13.3: + +Scrapy 2.13.3 (2025-07-02) +-------------------------- + +- Changed the values for :setting:`DOWNLOAD_DELAY` (from ``0`` to ``1``) and + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (from ``8`` to ``1``) in the + default project template. + (:issue:`6597`, :issue:`6918`, :issue:`6923`) + +- Improved :class:`scrapy.core.engine.ExecutionEngine` logic related to + initialization and exception handling, fixing several cases where the + spider would crash, hang or log an unhandled exception. + (:issue:`6783`, :issue:`6784`, :issue:`6900`, :issue:`6908`, :issue:`6910`, + :issue:`6911`) + +- Fixed a Windows issue with :ref:`feed exports ` using + :class:`scrapy.extensions.feedexport.FileFeedStorage` that caused the file + to be created on the wrong drive. + (:issue:`6894`, :issue:`6897`) + +- Allowed running tests with Twisted 25.5.0+ again. Pytest 8.4.1+ is now + required for running tests in non-pinned envs as support for the new + Twisted version was added in that version. + (:issue:`6893`) + +- Fixed running tests with lxml 6.0.0+. + (:issue:`6919`) + +- Added a deprecation notice for + ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` to :ref:`the Scrapy + 2.11.2 release notes `. + (:issue:`6926`) + +- Updated :ref:`contribution docs ` to refer to ruff_ + instead of black_. + (:issue:`6903`) + +- Added ``.venv/`` and ``.vscode/`` to ``.gitignore``. + (:issue:`6901`, :issue:`6907`) + + +.. _release-2.13.2: + +Scrapy 2.13.2 (2025-06-09) +-------------------------- + +- Fixed a bug introduced in Scrapy 2.13.0 that caused results of request + errbacks to be ignored when the errback was called because of a downloader + error. + (:issue:`6861`, :issue:`6863`) + +- Added a note about the behavior change of + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to its docs and + to the "Backward-incompatible changes" section of :ref:`the Scrapy 2.13.0 + release notes `. + (:issue:`6866`) + +- Improved the message in the exception raised by + :func:`scrapy.utils.test.get_reactor_settings` when there is no reactor + installed. + (:issue:`6866`) + +- Updated the :class:`scrapy.crawler.CrawlerRunner` examples in + :ref:`topics-practices` to install the reactor explicitly, to fix + reactor-related errors with Scrapy 2.13.0 and later. + (:issue:`6865`) + +- Fixed ``scrapy fetch`` not working with scrapy-poet_. + (:issue:`6872`) + +- Fixed an exception produced by :class:`scrapy.core.engine.ExecutionEngine` + when it's closed before being fully initialized. + (:issue:`6857`, :issue:`6867`) + +- Improved the README, updated the Scrapy logo in it. + (:issue:`6831`, :issue:`6833`, :issue:`6839`) + +- Restricted the Twisted version used in tests to below 25.5.0, as some tests + fail with 25.5.0. + (:issue:`6878`, :issue:`6882`) + +- Updated type hints for Twisted 25.5.0 changes. + (:issue:`6882`) + +- Removed the old artwork. + (:issue:`6874`) + + +.. _release-2.13.1: + +Scrapy 2.13.1 (2025-05-28) +-------------------------- + +- Give callback requests precedence over start requests when priority values + are the same. + + This makes changes from 2.13.0 to start request handling more intuitive and + backward compatible. For scenarios where all requests have the same + priorities, in 2.13.0 all start requests were sent before the first + callback request. In 2.13.1, same as in 2.12 and lower, start requests are + only sent when there are not enough pending callback requests to reach + concurrency limits. + + (:issue:`6828`) + +- Added a deepwiki_ badge to the README. (:issue:`6793`) + + .. _deepwiki: https://deepwiki.com/scrapy/scrapy + +- Fixed a typo in the code example of :ref:`start-requests-lazy`. + (:issue:`6812`, :issue:`6815`) + +- Fixed a typo in the :ref:`coroutine-support` section of the documentation. + (:issue:`6822`) + +- Made this page more prominently listed in PyPI project links. + (:issue:`6826`) + + +.. _release-2.13.0: + +Scrapy 2.13.0 (2025-05-08) +-------------------------- + +Highlights: + +- The asyncio reactor is now enabled by default + +- Replaced ``start_requests()`` (sync) with :meth:`~scrapy.Spider.start` + (async) and changed how it is iterated + +- Added the :reqmeta:`allow_offsite` request meta key + +- Spider middlewares that don't support asynchronous spider output are + deprecated + +- Added a base class for :ref:`universal spider middlewares + ` + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for PyPy 3.9. + (:issue:`6613`) + +- Added support for PyPy 3.11. + (:issue:`6697`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The default value of the :setting:`TWISTED_REACTOR` setting was changed + from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. This value + was used in newly generated projects since Scrapy 2.7.0 but now existing + projects that don't explicitly set this setting will also use the asyncio + reactor. You can :ref:`change this setting in your project + ` to use a different reactor. + (:issue:`6659`, :issue:`6713`) + +- The iteration of start requests and items no longer stops once there are + requests in the scheduler, and instead runs continuously until all start + requests have been scheduled. + + To reproduce the previous behavior, see :ref:`start-requests-lazy`. + (:issue:`6729`) + +- An unhandled exception from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.open_spider` method of a + :ref:`spider middleware ` no longer stops the + crawl. + (:issue:`6729`) + +- In ``scrapy.core.engine.ExecutionEngine``: + + - The second parameter of ``open_spider()``, ``start_requests``, has been + removed. The start requests are determined by the ``spider`` parameter + instead (see :meth:`~scrapy.Spider.start`). + + - The ``slot`` attribute has been renamed to ``_slot`` and should not be + used. + + (:issue:`6729`) + +- In ``scrapy.core.engine``, the ``Slot`` class has been renamed to ``_Slot`` + and should not be used. + (:issue:`6729`) + +- The ``slot`` :ref:`telnet variable ` has been removed. + (:issue:`6729`) + +- In ``scrapy.core.spidermw.SpiderMiddlewareManager``, + ``process_start_requests()`` has been replaced by ``process_start()``. + (:issue:`6729`) + +- The ``scrape_func`` callable passed to + ``scrapy.core.spidermw.SpiderMiddlewareManager.scrape_response()`` is now + called with 2 parameters, ``response`` and ``request``, instead of 3, and + must return a :class:`~twisted.internet.defer.Deferred` instead of an + iterable. + (:issue:`6787`) + +- The now-deprecated ``start_requests()`` method, when it returns an iterable + instead of being defined as a generator, is now executed *after* the + :ref:`scheduler ` instance has been created. + (:issue:`6729`) + +- When using :setting:`JOBDIR`, :ref:`start requests ` are + now serialized into their own, ``s``-suffixed priority folders. You can set + :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` or ``""`` to change that, + but the side effects may be undesirable. See + :setting:`SCHEDULER_START_DISK_QUEUE` for details. + (:issue:`6729`) + +- The URL length limit, set by the :setting:`URLLENGTH_LIMIT` setting, is now + also enforced for start requests. + (:issue:`6777`) + +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. This shouldn't affect normal Scrapy use cases, but it may affect + 3rd-party test suites that use Scrapy internals such as + :class:`~scrapy.crawler.Crawler` and don't install a reactor explicitly. If + you are affected by this change, you most likely need to install the + reactor before running Scrapy code that expects it to be installed. + (:issue:`6732`, :issue:`6735`) + +- The ``from_settings()`` method of + :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, + deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation + period (this was needed because after the introduction of the + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and + switching built-in spider middlewares to it those middlewares need the + :class:`~scrapy.crawler.Crawler` instance at run time). Please use + ``from_crawler()`` instead. + (:issue:`6693`) + +- ``scrapy.utils.url.escape_ajax()`` is no longer called when a + :class:`~scrapy.Request` instance is created. It was only useful for + websites supporting the ``_escaped_fragment_`` feature which most modern + websites don't support. If you still need this you can modify the URLs + before passing them to :class:`~scrapy.Request`. + (:issue:`6523`, :issue:`6651`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed old deprecated name aliases for some signals: + + - ``stats_spider_opened`` (use ``spider_opened`` instead) + + - ``stats_spider_closing`` and ``stats_spider_closed`` (use + ``spider_closed`` instead) + + - ``item_passed`` (use ``item_scraped`` instead) + + - ``request_received`` (use ``request_scheduled`` instead) + + (:issue:`6654`, :issue:`6655`) + +Deprecations +~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider` is deprecated, + use :meth:`~scrapy.Spider.start` instead, or both to maintain support for + lower Scrapy versions. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``process_start_requests()`` method of :ref:`spider middlewares + ` is deprecated, use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``__init__`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) should now support a keyword-only + ``start_queue_cls`` parameter. + (:issue:`6752`) + +- Spider middlewares that don't support asynchronous spider output are + deprecated. The async iterable downgrading feature, needed for using such + middlewares with asynchronous callbacks and with other spider middlewares + that produce asynchronous iterables, is also deprecated. Please update all + such middlewares to support asynchronous spider output. (:issue:`6664`) + +- Functions that were imported from :mod:`w3lib.url` and re-exported in + :mod:`scrapy.utils.url` are now deprecated, you should import them from + :mod:`w3lib.url` directly. They are: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + (:issue:`4577`, :issue:`6583`, :issue:`6586`) + +- HTTP/1.0 support code is deprecated. It was disabled by default and + couldn't be used together with HTTP/1.1. If you still need it, you should + write your own download handler or copy the code from Scrapy. The + deprecations include: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPClientFactory`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPPageGetter`` + + - Overriding + ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory.getContext()`` + + (:issue:`6634`) + +- The following modules and functions used only in tests are deprecated: + + - the ``scrapy.utils.testproc`` module + + - the ``scrapy.utils.testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + If you need to use them in your tests or code, you can copy the code from Scrapy. + (:issue:`6696`) + +- ``scrapy.utils.test.TestSpider`` is deprecated. If you need an empty spider + class you can use :class:`scrapy.utils.spider.DefaultSpider` or create your + own subclass of :class:`scrapy.Spider`. + (:issue:`6678`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` is + deprecated. It was disabled by default and isn't useful for most of the + existing websites. + (:issue:`6523`, :issue:`6651`, :issue:`6656`) + +- ``scrapy.utils.url.escape_ajax()`` is deprecated. + (:issue:`6523`, :issue:`6651`) + +- ``scrapy.spiders.init.InitSpider`` is deprecated. If you find it useful, + you can copy its code from Scrapy. + (:issue:`6708`, :issue:`6714`) + +- ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use + :func:`scrapy.utils.versions.get_versions` instead. + (:issue:`6582`) + +- ``BaseDupeFilter.log()`` is deprecated. It does nothing and shouldn't be + called. + (:issue:`4151`) + +- Passing the ``spider`` argument to the following methods of + :class:`~scrapy.core.scraper.Scraper` is deprecated: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`6764`) + +New features +~~~~~~~~~~~~ + +- You can now yield the start requests and items of a spider from the + :meth:`~scrapy.Spider.start` spider method and from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` spider + middleware method, both :term:`asynchronous generators `. + + This makes it possible to use asynchronous code to generate those start + requests and items, e.g. reading them from a queue service or database + using an asynchronous client, without workarounds. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- Start requests are now :ref:`scheduled ` as soon as + possible. + + As a result, their :attr:`~scrapy.Request.priority` is now taken into + account as soon as :setting:`CONCURRENT_REQUESTS` is reached. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- :class:`Crawler.signals ` has a new + :meth:`~scrapy.signalmanager.SignalManager.wait_for` method. + (:issue:`6729`) + +- Added a new :signal:`scheduler_empty` signal. + (:issue:`6729`) + +- Added new settings: :setting:`SCHEDULER_START_DISK_QUEUE` and + :setting:`SCHEDULER_START_MEMORY_QUEUE`. + (:issue:`6729`) + +- Added :class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware`, which + sets :reqmeta:`is_start_request` to ``True`` on :ref:`start requests + `. + (:issue:`6729`) + +- Exposed a new method of :class:`Crawler.engine + `: + :meth:`~scrapy.core.engine.ExecutionEngine.needs_backout`. + (:issue:`6729`) + +- Added the :reqmeta:`allow_offsite` request meta key that can be used + instead of the more general :attr:`~scrapy.Request.dont_filter` request + attribute to skip processing of the request by + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` (but not + by other code that checks :attr:`~scrapy.Request.dont_filter`). + (:issue:`3690`, :issue:`6151`, :issue:`6366`) + +- Added an optional base class for spider middlewares, + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which can be + helpful for writing :ref:`universal spider middlewares + ` without boilerplate and code duplication. + The built-in spider middlewares now inherit from this class. + (:issue:`6693`, :issue:`6777`) + +- :ref:`Scrapy add-ons ` can now define a class method called + ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings + `. + (:issue:`6544`, :issue:`6568`) + +- Added :ref:`helpers ` for modifying :ref:`component + priority dictionary ` settings. + (:issue:`6614`) + +- Responses that use an unknown/unsupported encoding now produce a warning. + If Scrapy knows that installing an additional package (such as brotli_) + will allow decoding the response, that will be mentioned in the warning. + (:issue:`4697`, :issue:`6618`) + +- Added the ``spider_exceptions/count`` stat which tracks the total count of + exceptions (tracked also by per-type ``spider_exceptions/*`` stats). + (:issue:`6739`, :issue:`6740`) + +- Added the :setting:`DEFAULT_DROPITEM_LOG_LEVEL` setting and the + :attr:`scrapy.exceptions.DropItem.log_level` attribute that allow + customizing the log level of the message that is logged when an item is + dropped. + (:issue:`6603`, :issue:`6608`) + +- Added support for the ``-b, --cookie`` curl argument to + :meth:`scrapy.Request.from_curl`. + (:issue:`6684`) + +- Added the :setting:`LOG_VERSIONS` setting that allows customizing the + list of software whose versions are logged when the spider starts. + (:issue:`6582`) + +- Added the :setting:`WARN_ON_GENERATOR_RETURN_VALUE` setting that allows + disabling run time analysis of callback code used to warn about incorrect + ``return`` statements in generator-based callbacks. You may need to disable + this setting if this analysis breaks on your callback code. + (:issue:`6731`, :issue:`6738`) + +Improvements +~~~~~~~~~~~~ + +- Removed or postponed some calls of :func:`itemadapter.is_item` to increase + performance. + (:issue:`6719`) + +- Improved the error message when running a ``scrapy`` command that requires + a project (such as ``scrapy crawl``) outside of a project directory. + (:issue:`2349`, :issue:`3426`) + +- Added an empty :setting:`ADDONS` setting to the ``settings.py`` template + for new projects. + (:issue:`6587`) + +Bug fixes +~~~~~~~~~ + +- Yielding an item from :meth:`Spider.start ` or from + :meth:`SpiderMiddleware.process_start + ` no longer delays + the next iteration of starting requests and items by up to 5 seconds. + (:issue:`6729`) + +- Fixed calculation of ``items_per_minute`` and ``responses_per_minute`` + stats. + (:issue:`6599`) + +- Fixed an error initializing + :class:`scrapy.extensions.feedexport.GCSFeedStorage`. + (:issue:`6617`, :issue:`6628`) + +- Fixed an error running ``scrapy bench``. + (:issue:`6632`, :issue:`6633`) + +- Fixed duplicated log messages about the reactor and the event loop. + (:issue:`6636`, :issue:`6657`) + +- Fixed resolving type annotations of ``SitemapSpider._parse_sitemap()`` at + run time, required by tools such as scrapy-poet_. + (:issue:`6665`, :issue:`6671`) + + .. _scrapy-poet: https://github.com/scrapinghub/scrapy-poet + +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. + (:issue:`6732`, :issue:`6735`) + +- Restored support for the ``x-gzip`` content encoding. + (:issue:`6618`) + +Documentation +~~~~~~~~~~~~~ + +- Documented the setting values set in the default project template. + (:issue:`6762`, :issue:`6775`) + +- Improved the docs about asynchronous iterable support in spider + middlewares. (:issue:`6688`) + +- Improved the :ref:`docs ` about using + :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based + code and included a list of such APIs. + (:issue:`6677`, :issue:`6734`, :issue:`6776`) + +- Improved the :ref:`contribution docs `. + (:issue:`6561`, :issue:`6575`) + +- Removed the ``Splash`` recommendation from the :ref:`headless browser + ` suggestion. We no longer recommend using + ``Splash`` and recommend using other headless browser solutions instead. + (:issue:`6642`, :issue:`6701`) + +- Added the dark mode to the HTML documentation. + (:issue:`6653`) + +- Other documentation improvements and fixes. + (:issue:`4151`, + :issue:`6526`, + :issue:`6620`, + :issue:`6621`, + :issue:`6622`, + :issue:`6623`, + :issue:`6624`, + :issue:`6721`, + :issue:`6723`, + :issue:`6780`) + +Packaging +~~~~~~~~~ + +- Switched from ``setup.py`` to ``pyproject.toml``. + (:issue:`6514`, :issue:`6547`) + +- Switched the build backend from setuptools_ to hatchling_. + (:issue:`6771`) + + .. _hatchling: https://pypi.org/project/hatchling/ + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Replaced most linters with ruff_. + (:issue:`6565`, + :issue:`6576`, + :issue:`6577`, + :issue:`6581`, + :issue:`6584`, + :issue:`6595`, + :issue:`6601`, + :issue:`6631`) + + .. _ruff: https://docs.astral.sh/ruff/ + +- Improved accuracy and performance of collecting test coverage. + (:issue:`6255`, :issue:`6610`) + +- Fixed an error that prevented running tests from directories other than the + top level source directory. + (:issue:`6567`) + +- Reduced the amount of ``mockserver`` calls in tests to improve the overall + test run time. + (:issue:`6637`, :issue:`6648`) + +- Fixed tests that were running the same test code more than once. + (:issue:`6646`, :issue:`6647`, :issue:`6650`) + +- Refactored tests to use more ``pytest`` features instead of ``unittest`` + ones where possible. + (:issue:`6678`, + :issue:`6680`, + :issue:`6695`, + :issue:`6699`, + :issue:`6700`, + :issue:`6702`, + :issue:`6709`, + :issue:`6710`, + :issue:`6711`, + :issue:`6712`, + :issue:`6725`) + +- Type hints improvements and fixes. + (:issue:`6578`, + :issue:`6579`, + :issue:`6593`, + :issue:`6605`, + :issue:`6694`) + +- CI and test improvements and fixes. + (:issue:`5360`, + :issue:`6271`, + :issue:`6547`, + :issue:`6560`, + :issue:`6602`, + :issue:`6607`, + :issue:`6609`, + :issue:`6613`, + :issue:`6619`, + :issue:`6626`, + :issue:`6679`, + :issue:`6703`, + :issue:`6704`, + :issue:`6716`, + :issue:`6720`, + :issue:`6722`, + :issue:`6724`, + :issue:`6741`, + :issue:`6743`, + :issue:`6766`, + :issue:`6770`, + :issue:`6772`, + :issue:`6773`) + +- Code cleanups. + (:issue:`6600`, + :issue:`6606`, + :issue:`6635`, + :issue:`6764`) + + +.. _release-2.12.0: + +Scrapy 2.12.0 (2024-11-18) +-------------------------- + +Highlights: + +- Dropped support for Python 3.8, added support for Python 3.13 + +- ``scrapy.Spider.start_requests()`` can now yield items + +- Added :class:`~scrapy.http.JsonResponse` + +- Added :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.8. + (:issue:`6466`, :issue:`6472`) + +- Added support for Python 3.13. + (:issue:`6166`) + +- Minimum versions increased for these dependencies: + + - Twisted_: 18.9.0 → 21.7.0 + + - cryptography_: 36.0.0 → 37.0.0 + + - pyOpenSSL_: 21.0.0 → 22.0.0 + + - lxml_: 4.4.1 → 4.6.0 + +- Removed ``setuptools`` from the dependency list. + (:issue:`6487`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- User-defined cookies for HTTPS requests will have the ``secure`` flag set + to ``True`` unless it's set to ``False`` explicitly. This is important when + these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP + URL. + (:issue:`6357`) + +- The Reppy-based ``robots.txt`` parser, + ``scrapy.robotstxt.ReppyRobotParser``, was removed, as it doesn't support + Python 3.9+. + (:issue:`5230`, :issue:`6099`, :issue:`6499`) + +- The initialization API of :class:`scrapy.pipelines.media.MediaPipeline` and + its subclasses was improved and it's possible that some previously working + usage scenarios will no longer work. It can only affect you if you define + custom subclasses of ``MediaPipeline`` or create instances of these + pipelines via ``from_settings()`` or ``__init__()`` calls instead of + ``from_crawler()`` calls. + + Previously, ``MediaPipeline.from_crawler()`` called the ``from_settings()`` + method if it existed or the ``__init__()`` method otherwise, and then did + some additional initialization using the ``crawler`` instance. If the + ``from_settings()`` method existed (like in ``FilesPipeline``) it called + ``__init__()`` to create the instance. It wasn't possible to override + ``from_crawler()`` without calling ``MediaPipeline.from_crawler()`` from it + which, in turn, couldn't be called in some cases (including subclasses of + ``FilesPipeline``). + + Now, in line with the general usage of ``from_crawler()`` and + ``from_settings()`` and the deprecation of the latter the recommended + initialization order is the following one: + + - All ``__init__()`` methods should take a ``crawler`` argument. If they + also take a ``settings`` argument they should ignore it, using + ``crawler.settings`` instead. When they call ``__init__()`` of the base + class they should pass the ``crawler`` argument to it too. + - A ``from_settings()`` method shouldn't be defined. Class-specific + initialization code should go into either an overridden ``from_crawler()`` + method or into ``__init__()``. + - It's now possible to override ``from_crawler()`` and it's not necessary + to call ``MediaPipeline.from_crawler()`` in it if other recommendations + were followed. + - If pipeline instances were created with ``from_settings()`` or + ``__init__()`` calls (which wasn't supported even before, as it missed + important initialization code), they should now be created with + ``from_crawler()`` calls. + + (:issue:`6540`) + +- The ``response_body`` argument of :meth:`ImagesPipeline.convert_image + ` is now + positional-only, as it was changed from optional to required. + (:issue:`6500`) + +- The ``convert`` argument of :func:`scrapy.utils.conf.build_component_list` + is now positional-only, as the preceding argument (``custom``) was removed. + (:issue:`6500`) + +- The ``overwrite_output`` argument of + :func:`scrapy.utils.conf.feed_process_params_from_cli` is now + positional-only, as the preceding argument (``output_format``) was removed. + (:issue:`6500`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed the ``scrapy.utils.request.request_fingerprint()`` function, + deprecated in Scrapy 2.7.0. + (:issue:`6212`, :issue:`6213`) + +- Removed support for value ``"2.6"`` of setting + ``REQUEST_FINGERPRINTER_IMPLEMENTATION``, deprecated in Scrapy 2.7.0. + (:issue:`6212`, :issue:`6213`) + +- :class:`~scrapy.dupefilters.RFPDupeFilter` subclasses now require + supporting the ``fingerprinter`` parameter in their ``__init__`` method, + introduced in Scrapy 2.7.0. + (:issue:`6102`, :issue:`6113`) + +- Removed the ``scrapy.downloadermiddlewares.decompression`` module, + deprecated in Scrapy 2.7.0. + (:issue:`6100`, :issue:`6113`) + +- Removed the ``scrapy.utils.response.response_httprepr()`` function, + deprecated in Scrapy 2.6.0. + (:issue:`6111`, :issue:`6116`) + +- Spiders with spider-level HTTP authentication, i.e. with the ``http_user`` + or ``http_pass`` attributes, must now define ``http_auth_domain`` as well, + which was introduced in Scrapy 2.5.1. + (:issue:`6103`, :issue:`6113`) + +- :ref:`Media pipelines ` methods ``file_path()``, + ``file_downloaded()``, ``get_images()``, ``image_downloaded()``, + ``media_downloaded()``, ``media_to_download()``, and ``thumb_path()`` must + now support an ``item`` parameter, added in Scrapy 2.4.0. + (:issue:`6107`, :issue:`6113`) + +- The ``__init__()`` and ``from_crawler()`` methods of :ref:`feed storage + backend classes ` must now support the keyword-only + ``feed_options`` parameter, introduced in Scrapy 2.4.0. + (:issue:`6105`, :issue:`6113`) + +- Removed the ``scrapy.loader.common`` and ``scrapy.loader.processors`` + modules, deprecated in Scrapy 2.3.0. + (:issue:`6106`, :issue:`6113`) + +- Removed the ``scrapy.utils.misc.extract_regex()`` function, deprecated in + Scrapy 2.3.0. + (:issue:`6106`, :issue:`6113`) + +- Removed the ``scrapy.http.JSONRequest`` class, replaced with + ``JsonRequest`` in Scrapy 1.8.0. + (:issue:`6110`, :issue:`6113`) + +- ``scrapy.utils.log.logformatter_adapter`` no longer supports missing + ``args``, ``level``, or ``msg`` parameters, and no longer supports a + ``format`` parameter, all scenarios that were deprecated in Scrapy 1.0.0. + (:issue:`6109`, :issue:`6116`) + +- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that + does not implement the ``ISpiderLoader`` interface + will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at + run time. Non-compliant classes have been triggering a deprecation warning + since Scrapy 1.0.0. + (:issue:`6101`, :issue:`6113`) + +- Removed the ``--output-format``/``-t`` command line option, deprecated in + Scrapy 2.1.0. ``-O :`` should be used instead. + (:issue:`6500`) + +- Running :meth:`~scrapy.crawler.Crawler.crawl` more than once on the same + :class:`~scrapy.crawler.Crawler` instance, deprecated in Scrapy 2.11.0, now + raises an exception. + (:issue:`6500`) + +- Subclassing + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + without support for the ``crawler`` argument in ``__init__()`` and without + a custom ``from_crawler()`` method, deprecated in Scrapy 2.5.0, is no + longer allowed. + (:issue:`6500`) + +- Removed the ``EXCEPTIONS_TO_RETRY`` attribute of + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`, deprecated in + Scrapy 2.10.0. + (:issue:`6500`) + +- Removed support for :ref:`S3 feed exports ` without + the boto3_ package installed, deprecated in Scrapy 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.extensions.feedexport._FeedSlot`` class, deprecated in + Scrapy 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.pipelines.images.NoimagesDrop`` exception, deprecated + in Scrapy 2.8.0. + (:issue:`6500`) + +- The ``response_body`` argument of :meth:`ImagesPipeline.convert_image + ` is now required, + not passing it was deprecated in Scrapy 2.8.0. + (:issue:`6500`) + +- Removed the ``custom`` argument of + :func:`scrapy.utils.conf.build_component_list`, deprecated in Scrapy + 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.utils.reactor.get_asyncio_event_loop_policy()`` + function, deprecated in Scrapy 2.9.0. Use :func:`asyncio.get_event_loop` + and related standard library functions instead. + (:issue:`6500`) + +Deprecations +~~~~~~~~~~~~ + +- The ``from_settings()`` methods of the :ref:`Scrapy components + ` that have them are now deprecated. ``from_crawler()`` + should now be used instead. Affected components: + + - :class:`scrapy.dupefilters.RFPDupeFilter` + - :class:`scrapy.mail.MailSender` + - :class:`scrapy.middleware.MiddlewareManager` + - :class:`scrapy.core.downloader.contextfactory.ScrapyClientContextFactory` + - :class:`scrapy.pipelines.files.FilesPipeline` + - :class:`scrapy.pipelines.images.ImagesPipeline` + - :class:`scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` + + (:issue:`6540`) + +- It's now deprecated to have a ``from_settings()`` method but no + ``from_crawler()`` method in 3rd-party :ref:`Scrapy components + `. You can define a simple ``from_crawler()`` method + that calls ``cls.from_settings(crawler.settings)`` to fix this if you don't + want to refactor the code. Note that if you have a ``from_crawler()`` + method Scrapy will not call the ``from_settings()`` method so the latter + can be removed. + (:issue:`6540`) + +- The initialization API of :class:`scrapy.pipelines.media.MediaPipeline` and + its subclasses was improved and some old usage scenarios are now deprecated + (see also the "Backward-incompatible changes" section). Specifically: + + - It's deprecated to define an ``__init__()`` method that doesn't take a + ``crawler`` argument. + - It's deprecated to call an ``__init__()`` method without passing a + ``crawler`` argument. If it's passed, it's also deprecated to pass a + ``settings`` argument, which will be ignored anyway. + - Calling ``from_settings()`` is deprecated, use ``from_crawler()`` + instead. + - Overriding ``from_settings()`` is deprecated, override ``from_crawler()`` + instead. + + (:issue:`6540`) + +- The ``REQUEST_FINGERPRINTER_IMPLEMENTATION`` setting is now deprecated. + (:issue:`6212`, :issue:`6213`) + +- The ``scrapy.utils.misc.create_instance()`` function is now deprecated, use + :func:`scrapy.utils.misc.build_from_crawler` instead. + (:issue:`5523`, :issue:`5884`, :issue:`6162`, :issue:`6169`, :issue:`6540`) + +- ``scrapy.core.downloader.Downloader._get_slot_key()`` is deprecated, use + :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. + (:issue:`6340`, :issue:`6352`) + +- ``scrapy.utils.defer.process_chain_both()`` is now deprecated. + (:issue:`6397`) + +- ``scrapy.twisted_version`` is now deprecated, you should instead use + :attr:`twisted.version` directly (but note that it's an + ``incremental.Version`` object, not a tuple). + (:issue:`6509`, :issue:`6512`) + +- ``scrapy.utils.python.flatten()`` and ``scrapy.utils.python.iflatten()`` + are now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.python.equal_attributes()`` is now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.request.request_authenticate()`` is now deprecated, you + should instead just set the ``Authorization`` header directly. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.serialize.ScrapyJSONDecoder`` is now deprecated, it didn't + contain any code since Scrapy 1.0.0. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.test.assert_samelines()`` is now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.extensions.feedexport.build_storage()`` is now deprecated. You can + instead call the builder callable directly. + (:issue:`6540`) + +New features +~~~~~~~~~~~~ + +- ``scrapy.Spider.start_requests()`` can now yield items. + (:issue:`5289`, :issue:`6417`) + + .. note:: Some spider middlewares may need to be updated for Scrapy 2.12 + support before you can use them in combination with the ability to + yield items from ``start_requests()``. + +- Added a new :class:`~scrapy.http.Response` subclass, + :class:`~scrapy.http.JsonResponse`, for responses with a `JSON MIME type + `_. + (:issue:`6069`, :issue:`6171`, :issue:`6174`) + +- The :class:`~scrapy.extensions.logstats.LogStats` extension now adds + ``items_per_minute`` and ``responses_per_minute`` to the :ref:`stats + ` when the spider closes. + (:issue:`4110`, :issue:`4111`) + +- Added :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` which allows closing the + spider if no items were scraped in a set amount of time. + (:issue:`6434`) + +- User-defined cookies can now include the ``secure`` field. + (:issue:`6357`) + +- Added component getters to :class:`~scrapy.crawler.Crawler`: + :meth:`~scrapy.crawler.Crawler.get_addon`, + :meth:`~scrapy.crawler.Crawler.get_downloader_middleware`, + :meth:`~scrapy.crawler.Crawler.get_extension`, + :meth:`~scrapy.crawler.Crawler.get_item_pipeline`, + :meth:`~scrapy.crawler.Crawler.get_spider_middleware`. + (:issue:`6181`) + +- Slot delay updates by the :ref:`AutoThrottle extension + ` based on response latencies can now be disabled for + specific requests via the :reqmeta:`autothrottle_dont_adjust_delay` meta + key. + (:issue:`6246`, :issue:`6527`) + +- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, + :class:`~scrapy.spiderloader.SpiderLoader` does not raise + :exc:`SyntaxError` but emits a warning instead. + (:issue:`6483`, :issue:`6484`) + +- Added support for multiple-compressed responses (ones with several + encodings in the ``Content-Encoding`` header). + (:issue:`5143`, :issue:`5964`, :issue:`6063`) + +- Added support for multiple standard values in :setting:`REFERRER_POLICY`. + (:issue:`6381`) + +- Added support for brotlicffi_ (previously named brotlipy_). brotli_ is + still recommended but only brotlicffi_ works on PyPy. + (:issue:`6263`, :issue:`6269`) + + .. _brotlicffi: https://github.com/python-hyper/brotlicffi + +- Added :class:`~scrapy.contracts.default.MetadataContract` that sets the + request meta. + (:issue:`6468`, :issue:`6469`) + +Improvements +~~~~~~~~~~~~ + +- Extended the list of file extensions that + :class:`LinkExtractor ` + ignores by default. + (:issue:`6074`, :issue:`6125`) + +- :func:`scrapy.utils.httpobj.urlparse_cached` is now used in more places + instead of :func:`urllib.parse.urlparse`. + (:issue:`6228`, :issue:`6229`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.pipelines.media.MediaPipeline` is now an abstract class and + its methods that were expected to be overridden in subclasses are now + abstract methods. + (:issue:`6365`, :issue:`6368`) + +- Fixed handling of invalid ``@``-prefixed lines in contract extraction. + (:issue:`6383`, :issue:`6388`) + +- Importing ``scrapy.extensions.telnet`` no longer installs the default + reactor. + (:issue:`6432`) + +- Reduced log verbosity for dropped requests that was increased in 2.11.2. + (:issue:`6433`, :issue:`6475`) + +Documentation +~~~~~~~~~~~~~ + +- Added ``SECURITY.md`` that documents the security policy. + (:issue:`5364`, :issue:`6051`) + +- Example code for :ref:`running Scrapy from a script ` no + longer imports ``twisted.internet.reactor`` at the top level, which caused + problems with non-default reactors when this code was used unmodified. + (:issue:`6361`, :issue:`6374`) + +- Documented the :class:`~scrapy.extensions.spiderstate.SpiderState` + extension. + (:issue:`6278`, :issue:`6522`) + +- Other documentation improvements and fixes. + (:issue:`5920`, + :issue:`6094`, + :issue:`6177`, + :issue:`6200`, + :issue:`6207`, + :issue:`6216`, + :issue:`6223`, + :issue:`6317`, + :issue:`6328`, + :issue:`6389`, + :issue:`6394`, + :issue:`6402`, + :issue:`6411`, + :issue:`6427`, + :issue:`6429`, + :issue:`6440`, + :issue:`6448`, + :issue:`6449`, + :issue:`6462`, + :issue:`6497`, + :issue:`6506`, + :issue:`6507`, + :issue:`6524`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added ``py.typed``, in line with `PEP 561 + `_. + (:issue:`6058`, :issue:`6059`) + +- Fully covered the code with type hints (except for the most complicated + parts, mostly related to ``twisted.web.http`` and other Twisted parts + without type hints). + (:issue:`5989`, + :issue:`6097`, + :issue:`6127`, + :issue:`6129`, + :issue:`6130`, + :issue:`6133`, + :issue:`6143`, + :issue:`6191`, + :issue:`6268`, + :issue:`6274`, + :issue:`6275`, + :issue:`6276`, + :issue:`6279`, + :issue:`6325`, + :issue:`6326`, + :issue:`6333`, + :issue:`6335`, + :issue:`6336`, + :issue:`6337`, + :issue:`6341`, + :issue:`6353`, + :issue:`6356`, + :issue:`6370`, + :issue:`6371`, + :issue:`6384`, + :issue:`6385`, + :issue:`6387`, + :issue:`6391`, + :issue:`6395`, + :issue:`6414`, + :issue:`6422`, + :issue:`6460`, + :issue:`6466`, + :issue:`6472`, + :issue:`6494`, + :issue:`6498`, + :issue:`6516`) + +- Improved Bandit_ checks. + (:issue:`6260`, :issue:`6264`, :issue:`6265`) + +- Added pyupgrade_ to the ``pre-commit`` configuration. + (:issue:`6392`) + + .. _pyupgrade: https://github.com/asottile/pyupgrade + +- Added ``flake8-bugbear``, ``flake8-comprehensions``, ``flake8-debugger``, + ``flake8-docstrings``, ``flake8-string-format`` and + ``flake8-type-checking`` to the ``pre-commit`` configuration. + (:issue:`6406`, :issue:`6413`) + +- CI and test improvements and fixes. + (:issue:`5285`, + :issue:`5454`, + :issue:`5997`, + :issue:`6078`, + :issue:`6084`, + :issue:`6087`, + :issue:`6132`, + :issue:`6153`, + :issue:`6154`, + :issue:`6201`, + :issue:`6231`, + :issue:`6232`, + :issue:`6235`, + :issue:`6236`, + :issue:`6242`, + :issue:`6245`, + :issue:`6253`, + :issue:`6258`, + :issue:`6259`, + :issue:`6270`, + :issue:`6272`, + :issue:`6286`, + :issue:`6290`, + :issue:`6296` + :issue:`6367`, + :issue:`6372`, + :issue:`6403`, + :issue:`6416`, + :issue:`6435`, + :issue:`6489`, + :issue:`6501`, + :issue:`6504`, + :issue:`6511`, + :issue:`6543`, + :issue:`6545`) + +- Code cleanups. + (:issue:`6196`, + :issue:`6197`, + :issue:`6198`, + :issue:`6199`, + :issue:`6254`, + :issue:`6257`, + :issue:`6285`, + :issue:`6305`, + :issue:`6343`, + :issue:`6349`, + :issue:`6386`, + :issue:`6415`, + :issue:`6463`, + :issue:`6470`, + :issue:`6499`, + :issue:`6505`, + :issue:`6510`, + :issue:`6531`, + :issue:`6542`) + +Other +~~~~~ + +- Issue tracker improvements. (:issue:`6066`) + + +.. _release-2.11.2: + +Scrapy 2.11.2 (2024-05-14) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Redirects to non-HTTP protocols are no longer followed. Please, see the + `23j4-mw76-5v7h security advisory`_ for more information. (:issue:`457`) + + .. _23j4-mw76-5v7h security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-23j4-mw76-5v7h + +- The ``Authorization`` header is now dropped on redirects to a different + scheme (``http://`` or ``https://``) or port, even if the domain is the + same. Please, see the `4qqq-9vqf-3h3f security advisory`_ for more + information. + + .. _4qqq-9vqf-3h3f security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-4qqq-9vqf-3h3f + +- When using system proxy settings that are different for ``http://`` and + ``https://``, redirects to a different URL scheme will now also trigger the + corresponding change in proxy settings for the redirected request. Please, + see the `jm3v-qxmh-hxwv security advisory`_ for more information. + (:issue:`767`) + + .. _jm3v-qxmh-hxwv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-jm3v-qxmh-hxwv + +- :attr:`Spider.allowed_domains ` is now + enforced for all requests, and not only requests from spider callbacks. + (:issue:`1042`, :issue:`2241`, :issue:`6358`) + +- :func:`~scrapy.utils.iterators.xmliter_lxml` no longer resolves XML + entities. (:issue:`6265`) + +- defusedxml_ is now used to make + :class:`scrapy.http.request.rpc.XmlRpcRequest` more secure. + (:issue:`6250`, :issue:`6251`) + + .. _defusedxml: https://github.com/tiran/defusedxml + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` (a spider + middleware) is now deprecated and not enabled by default. The new + downloader middleware with the same functionality, + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, is enabled + instead. + (:issue:`2241`, :issue:`6358`) + + +Bug fixes +~~~~~~~~~ + +- Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in + favor of brotli_. (:issue:`6261`) + + .. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli + instead if you can. + +- Make :setting:`METAREFRESH_IGNORE_TAGS` ``["noscript"]`` by default. This + prevents + :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` from + following redirects that would not be followed by web browsers with + JavaScript enabled. (:issue:`6342`, :issue:`6347`) + +- During :ref:`feed export `, do not close the + underlying file from :ref:`built-in post-processing plugins + `. + (:issue:`5932`, :issue:`6178`, :issue:`6239`) + +- :class:`LinkExtractor ` + now properly applies the ``unique`` and ``canonicalize`` parameters. + (:issue:`3273`, :issue:`6221`) + +- Do not initialize the scheduler disk queue if :setting:`JOBDIR` is an empty + string. (:issue:`6121`, :issue:`6124`) + +- Fix :attr:`Spider.logger ` not logging custom extra + information. (:issue:`6323`, :issue:`6324`) + +- ``robots.txt`` files with a non-UTF-8 encoding no longer prevent parsing + the UTF-8-compatible (e.g. ASCII) parts of the document. + (:issue:`6292`, :issue:`6298`) + +- :meth:`scrapy.http.cookies.WrappedRequest.get_header` no longer raises an + exception if ``default`` is ``None``. + (:issue:`6308`, :issue:`6310`) + +- :class:`~scrapy.Selector` now uses + :func:`scrapy.utils.response.get_base_url` to determine the base URL of a + given :class:`~scrapy.http.Response`. (:issue:`6265`) + +- The :meth:`media_to_download` method of :ref:`media pipelines + ` now logs exceptions before stripping them. + (:issue:`5067`, :issue:`5068`) + +- When passing a callback to the :command:`parse` command, build the callback + callable with the right signature. + (:issue:`6182`) + +Documentation +~~~~~~~~~~~~~ + +- Add a FAQ entry about :ref:`creating blank requests `. + (:issue:`6203`, :issue:`6208`) + +- Document that :attr:`scrapy.Selector.type` can be ``"json"``. + (:issue:`6328`, :issue:`6334`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Make builds reproducible. (:issue:`5019`, :issue:`6322`) + +- Packaging and test fixes. + (:issue:`6286`, :issue:`6290`, :issue:`6312`, :issue:`6316`, :issue:`6344`) + + +.. _release-2.11.1: + +Scrapy 2.11.1 (2024-02-14) +-------------------------- + +Highlights: + +- Security bug fixes. + +- Support for Twisted >= 23.8.0. + +- Documentation improvements. + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Addressed `ReDoS vulnerabilities`_: + + - ``scrapy.utils.iterators.xmliter`` is now deprecated in favor of + :func:`~scrapy.utils.iterators.xmliter_lxml`, which + :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace with a prefix in the node name, and big files with + highly nested trees when using libxml2 2.7+. + + - Fixed regular expressions in the implementation of the + :func:`~scrapy.utils.response.open_in_browser` function. + + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. + + .. _ReDoS vulnerabilities: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS + .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 + +- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply + to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security + advisory`_ for more information. + + .. _7j7m-v7m3-jqm7 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-7j7m-v7m3-jqm7 + +- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, the + deprecated ``scrapy.downloadermiddlewares.decompression`` module has been + removed. + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The Twisted dependency is no longer restricted to < 23.8.0. (:issue:`6024`, + :issue:`6064`, :issue:`6142`) + +Bug fixes +~~~~~~~~~ + +- The OS signal handling code was refactored to no longer use private Twisted + functions. (:issue:`6024`, :issue:`6064`, :issue:`6112`) + +Documentation +~~~~~~~~~~~~~ + +- Improved documentation for :class:`~scrapy.crawler.Crawler` initialization + changes made in the 2.11.0 release. (:issue:`6057`, :issue:`6147`) + +- Extended documentation for :attr:`.Request.meta`. + (:issue:`5565`) + +- Fixed the :reqmeta:`dont_merge_cookies` documentation. (:issue:`5936`, + :issue:`6077`) + +- Added a link to Zyte's export guides to the :ref:`feed exports + ` documentation. (:issue:`6183`) + +- Added a missing note about backward-incompatible changes in + :class:`~scrapy.exporters.PythonItemExporter` to the 2.11.0 release notes. + (:issue:`6060`, :issue:`6081`) + +- Added a missing note about removing the deprecated + ``scrapy.utils.boto.is_botocore()`` function to the 2.8.0 release notes. + (:issue:`6056`, :issue:`6061`) + +- Other documentation improvements. (:issue:`6128`, :issue:`6144`, + :issue:`6163`, :issue:`6190`, :issue:`6192`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added Python 3.12 to the CI configuration, re-enabled tests that were + disabled when the pre-release support was added. (:issue:`5985`, + :issue:`6083`, :issue:`6098`) + +- Fixed a test issue on PyPy 7.3.14. (:issue:`6204`, :issue:`6205`) + + +.. _release-2.11.0: + +Scrapy 2.11.0 (2023-09-18) +-------------------------- + +Highlights: + +- Spiders can now modify :ref:`settings ` in their + :meth:`~scrapy.Spider.from_crawler` methods, e.g. based on :ref:`spider + arguments `. + +- Periodic logging of stats. + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Most of the initialization of :class:`scrapy.crawler.Crawler` instances is + now done in :meth:`~scrapy.crawler.Crawler.crawl`, so the state of + instances before that method is called is now different compared to older + Scrapy versions. We do not recommend using the + :class:`~scrapy.crawler.Crawler` instances before + :meth:`~scrapy.crawler.Crawler.crawl` is called. (:issue:`6038`) + +- :meth:`scrapy.Spider.from_crawler` is now called before the initialization + of various components previously initialized in + :meth:`scrapy.crawler.Crawler.__init__` and before the settings are + finalized and frozen. This change was needed to allow changing the settings + in :meth:`scrapy.Spider.from_crawler`. If you want to access the final + setting values and the initialized :class:`~scrapy.crawler.Crawler` + attributes in the spider code as early as possible you can do this in + ``scrapy.Spider.start_requests()`` or in a handler of the + :signal:`engine_started` signal. (:issue:`6038`) + - The :meth:`TextResponse.json ` method now requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or - UTF-32). + UTF-32). If you need to deal with JSON documents in an invalid encoding, + use ``json.loads(response.text)`` instead. (:issue:`6016`) - If you need to deal with JSON documents in an invalid encoding, use - ``json.loads(response.text)`` instead. +- :class:`~scrapy.exporters.PythonItemExporter` used the binary output by + default but it no longer does. (:issue:`6006`, :issue:`6007`) - (:issue:`5968`) +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed the binary export mode of + :class:`~scrapy.exporters.PythonItemExporter`, deprecated in Scrapy 1.1.0. + (:issue:`6006`, :issue:`6007`) + + .. note:: If you are using this Scrapy version on Scrapy Cloud with a stack + that includes an older Scrapy version and get a "TypeError: + Unexpected options: binary" error, you may need to add + ``scrapinghub-entrypoint-scrapy >= 0.14.1`` to your project + requirements or switch to a stack that includes Scrapy 2.11. + +- Removed the ``CrawlerRunner.spiders`` attribute, deprecated in Scrapy + 1.0.0, use :attr:`CrawlerRunner.spider_loader + ` instead. (:issue:`6010`) + +- The :func:`scrapy.utils.response.response_httprepr` function, deprecated in + Scrapy 2.6.0, has now been removed. (:issue:`6111`) + +Deprecations +~~~~~~~~~~~~ + +- Running :meth:`~scrapy.crawler.Crawler.crawl` more than once on the same + :class:`scrapy.crawler.Crawler` instance is now deprecated. (:issue:`1587`, + :issue:`6040`) + +New features +~~~~~~~~~~~~ + +- Spiders can now modify settings in their + :meth:`~scrapy.Spider.from_crawler` method, e.g. based on :ref:`spider + arguments `. (:issue:`1305`, :issue:`1580`, :issue:`2392`, + :issue:`3663`, :issue:`6038`) + +- Added the :class:`~scrapy.extensions.periodic_log.PeriodicLog` extension + which can be enabled to log stats and/or their differences periodically. + (:issue:`5926`) + +- Optimized the memory usage in :meth:`TextResponse.json + ` by removing unnecessary body decoding. + (:issue:`5968`, :issue:`6016`) + +- Links to ``.webp`` files are now ignored by :ref:`link extractors + `. (:issue:`6021`) + +Bug fixes +~~~~~~~~~ + +- Fixed logging enabled add-ons. (:issue:`6036`) + +- Fixed :class:`~scrapy.mail.MailSender` producing invalid message bodies + when the ``charset`` argument is passed to + :meth:`~scrapy.mail.MailSender.send`. (:issue:`5096`, :issue:`5118`) + +- Fixed an exception when accessing ``self.EXCEPTIONS_TO_RETRY`` from a + subclass of :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`. + (:issue:`6049`, :issue:`6050`) + +- :meth:`scrapy.settings.BaseSettings.getdictorlist`, used to parse + :setting:`FEED_EXPORT_FIELDS`, now handles tuple values. (:issue:`6011`, + :issue:`6013`) + +- Calls to ``datetime.utcnow()``, no longer recommended to be used, have been + replaced with calls to ``datetime.now()`` with a timezone. (:issue:`6014`) + +Documentation +~~~~~~~~~~~~~ + +- Updated a deprecated function call in a pipeline example. (:issue:`6008`, + :issue:`6009`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Extended typing hints. (:issue:`6003`, :issue:`6005`, :issue:`6031`, + :issue:`6034`) + +- Pinned brotli_ to 1.0.9 for the PyPy tests as 1.1.0 breaks them. + (:issue:`6044`, :issue:`6045`) + +- Other CI and pre-commit improvements. (:issue:`6002`, :issue:`6013`, + :issue:`6046`) .. _release-2.10.1: @@ -64,10 +3457,10 @@ Modified requirements Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` - instead of ``False``. In earlier Scrapy versions empty files were created - even when this setting was ``False`` (which was a bug that is now fixed), - so the new default should keep the old behavior. (:issue:`872`, +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, :issue:`5847`) Deprecation removals @@ -220,12 +3613,12 @@ Deprecations New features ~~~~~~~~~~~~ -- Settings correponding to :setting:`DOWNLOAD_DELAY`, +- Settings corresponding to :setting:`DOWNLOAD_DELAY`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) -- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors +- Added :meth:`.TextResponse.jmespath`, a shortcut for JMESPath selectors available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) - Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed` @@ -280,7 +3673,7 @@ Bug fixes (:issue:`5914`, :issue:`5917`) - Fixed an error breaking user handling of send failures in - :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) + :meth:`scrapy.mail.MailSender.send`. (:issue:`1611`, :issue:`5880`) Documentation ~~~~~~~~~~~~~ @@ -368,6 +3761,10 @@ Deprecation removals has now been removed. (:issue:`5719`) +- The ``scrapy.utils.boto.is_botocore()`` function, deprecated in Scrapy 2.4, + has now been removed. + (:issue:`5719`) + Deprecations ~~~~~~~~~~~~ @@ -401,7 +3798,7 @@ New features avoid confusion. (:issue:`5717`, :issue:`5722`, :issue:`5727`) -- The ``callback`` parameter of :class:`~scrapy.http.Request` can now be set +- The ``callback`` parameter of :class:`~scrapy.Request` can now be set to :func:`scrapy.http.request.NO_CALLBACK`, to distinguish it from ``None``, as the latter indicates that the default spider callback (:meth:`~scrapy.Spider.parse`) is to be used. @@ -695,7 +4092,7 @@ Bug fixes that does not match the asyncio event loop actually installed (:issue:`5529`). -- Fixed :meth:`Headers.getlist ` +- Fixed :meth:`Headers.getlist() ` returning only the last header (:issue:`5515`, :issue:`5526`). - Fixed :class:`LinkExtractor @@ -727,7 +4124,7 @@ Documentation (:issue:`3582`, :issue:`5432`). .. _Common Crawl: https://commoncrawl.org/ - .. _Google cache: http://www.googleguide.com/cached_pages.html + .. _Google cache: https://www.googleguide.com/cached_pages.html - The new :ref:`topics-components` topic covers enforcing requirements on Scrapy components, like :ref:`downloader middlewares @@ -898,17 +4295,17 @@ Highlights: Security bug fixes ~~~~~~~~~~~~~~~~~~ -- When a :class:`~scrapy.http.Request` object with cookies defined gets a - redirect response causing a new :class:`~scrapy.http.Request` object to be +- When a :class:`~scrapy.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.Request` object to be scheduled, the cookies defined in the original - :class:`~scrapy.http.Request` object are no longer copied into the new - :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object are no longer copied into the new + :class:`~scrapy.Request` object. If you manually set the ``Cookie`` header on a - :class:`~scrapy.http.Request` object and the domain name of the redirect + :class:`~scrapy.Request` object and the domain name of the redirect URL is not an exact match for the domain of the URL of the original - :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped - from the new :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.Request` object. The old behavior could be exploited by an attacker to gain access to your cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more @@ -921,10 +4318,10 @@ Security bug fixes ``example.com`` and any subdomain) by defining the shared domain suffix (e.g. ``example.com``) as the cookie domain when defining your cookies. See the documentation of the - :class:`~scrapy.http.Request` class for more information. + :class:`~scrapy.Request` class for more information. - When the domain of a cookie, either received in the ``Set-Cookie`` header - of a response or defined in a :class:`~scrapy.http.Request` object, is set + of a response or defined in a :class:`~scrapy.Request` object, is set to a `public suffix `_, the cookie is now ignored unless the cookie domain is the same as the request domain. @@ -940,7 +4337,7 @@ Modified requirements ~~~~~~~~~~~~~~~~~~~~~ - The h2_ dependency is now optional, only needed to - :ref:`enable HTTP/2 support `. (:issue:`5113`) + :ref:`enable HTTP/2 support `. (:issue:`5113`) .. _h2: https://pypi.org/project/h2/ @@ -975,7 +4372,7 @@ Backward-incompatible changes meet expectations, :exc:`TypeError` is now raised at startup time. Before, other exceptions would be raised at run time. (:issue:`3559`) -- The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects +- The ``_encoding`` field of serialized :class:`~scrapy.Request` objects is now named ``encoding``, in line with all other fields (:issue:`5130`) @@ -1005,7 +4402,7 @@ Deprecations - :mod:`scrapy.utils.reqser` is deprecated. (:issue:`5130`) - Instead of :func:`~scrapy.utils.reqser.request_to_dict`, use the new - :meth:`Request.to_dict ` method. + :meth:`.Request.to_dict` method. - Instead of :func:`~scrapy.utils.reqser.request_from_dict`, use the new :func:`scrapy.utils.request.request_from_dict` function. @@ -1047,6 +4444,9 @@ Deprecations Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider` first to set the :class:`~scrapy.Spider` object. +- :func:`scrapy.utils.response.response_httprepr` is now deprecated. + (:issue:`4972`) + New features ~~~~~~~~~~~~ @@ -1081,7 +4481,7 @@ New features (:setting:`AWS_SESSION_TOKEN`) and endpoint customization (:setting:`AWS_ENDPOINT_URL`). (:issue:`4998`, :issue:`5210`) - .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys + .. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html - New :setting:`LOG_FILE_APPEND` setting to allow truncating the log file. (:issue:`5279`) @@ -1107,9 +4507,9 @@ New features using ``queuelib`` 1.6.1 or later), the ``peek`` method raises :exc:`NotImplementedError`. -- :class:`~scrapy.http.Request` and :class:`~scrapy.http.Response` now have +- :class:`~scrapy.Request` and :class:`~scrapy.http.Response` now have an ``attributes`` attribute that makes subclassing easier. For - :class:`~scrapy.http.Request`, it also allows subclasses to work with + :class:`~scrapy.Request`, it also allows subclasses to work with :func:`scrapy.utils.request.request_from_dict`. (:issue:`1877`, :issue:`5130`, :issue:`5218`) @@ -1185,7 +4585,7 @@ Documentation - Provided better context and instructions to disable the :setting:`URLLENGTH_LIMIT` setting. (:issue:`5135`, :issue:`5250`) -- Documented that :ref:`reppy-parser` does not support Python 3.9+. +- Documented that Reppy parser does not support Python 3.9+. (:issue:`5226`, :issue:`5231`) - Documented :ref:`the scheduler component `. @@ -1227,7 +4627,7 @@ Documentation - ``quotes.toscrape.com`` references now use HTTPS instead of HTTP. (:issue:`5395`, :issue:`5396`) -- Added a link to `our Discord server `_ +- Added a link to `our Discord server `_ to :ref:`getting-help`. (:issue:`5421`, :issue:`5422`) - The pronunciation of the project name is now :ref:`officially @@ -1309,8 +4709,6 @@ Scrapy 2.5.1 (2021-10-05) need to upgrade scrapy-splash to a greater version for it to continue to work. -.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash - .. _release-2.5.0: @@ -1321,7 +4719,7 @@ Highlights: - Official Python 3.9 support -- Experimental :ref:`HTTP/2 support ` +- Experimental :ref:`HTTP/2 support ` - New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function to retry requests from spider callbacks @@ -1352,7 +4750,7 @@ Deprecations New features ~~~~~~~~~~~~ -- Experimental :ref:`HTTP/2 support ` through a new download handler +- Experimental :ref:`HTTP/2 support ` through a new download handler that can be assigned to the ``https`` protocol in the :setting:`DOWNLOAD_HANDLERS` setting. (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) @@ -1418,7 +4816,7 @@ Bug fixes with lower indentation than the following code. (:issue:`4477`, :issue:`4935`) -- The `Content-Length `_ +- The `Content-Length `_ header is no longer omitted from responses when using the default, HTTP/1.1 download handler (see :setting:`DOWNLOAD_HANDLERS`). (:issue:`5009`, :issue:`5034`, :issue:`5045`, :issue:`5057`, :issue:`5062`) @@ -1575,14 +4973,13 @@ Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again - discards cookies defined in :attr:`Request.headers - `. + discards cookies defined in :attr:`.Request.headers`. We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it was reported that the current implementation could break existing code. If you need to set cookies for a request, use the :class:`Request.cookies - ` parameter. + ` parameter. A future version of Scrapy will include a new, better implementation of the reverted bug fix. @@ -1703,16 +5100,16 @@ New features :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` or :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` - with a custom :class:`~scrapy.http.Request` object assigned to + with a custom :class:`~scrapy.Request` object assigned to :class:`response.request `: - The response is handled by the callback of that custom - :class:`~scrapy.http.Request` object, instead of being handled by the - callback of the original :class:`~scrapy.http.Request` object + :class:`~scrapy.Request` object, instead of being handled by the + callback of the original :class:`~scrapy.Request` object - - That custom :class:`~scrapy.http.Request` object is now sent as the + - That custom :class:`~scrapy.Request` object is now sent as the ``request`` argument to the :signal:`response_received` signal, instead - of the original :class:`~scrapy.http.Request` object + of the original :class:`~scrapy.Request` object (:issue:`4529`, :issue:`4632`) @@ -1766,7 +5163,7 @@ Bug fixes * The system file mode creation mask no longer affects the permissions of files generated using the :command:`startproject` command (:issue:`4722`) -* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names +* ``scrapy.utils.iterators.xmliter`` now supports namespaced node names (:issue:`861`, :issue:`4746`) * :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can @@ -1883,7 +5280,7 @@ New features * The :command:`parse` command now allows specifying an output file (:issue:`4317`, :issue:`4377`) -* :meth:`Request.from_curl ` and +* :meth:`.Request.from_curl` and :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support ``--data-raw`` (:issue:`4612`) @@ -1899,7 +5296,7 @@ Bug fixes :ref:`dataclass items ` and :ref:`attr.s items ` (:issue:`4667`, :issue:`4668`) -* :meth:`Request.from_curl ` and +* :meth:`.Request.from_curl` and :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request method to ``POST`` when a request body is specified and no request method is specified (:issue:`4612`) @@ -1918,7 +5315,7 @@ Documentation * Simplified the code example in :ref:`topics-loaders-dataclass` (:issue:`4652`) -.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT +.. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format Quality assurance @@ -1984,8 +5381,7 @@ Backward-incompatible changes Deprecations ~~~~~~~~~~~~ -* :meth:`TextResponse.body_as_unicode - ` is now deprecated, use +* ``TextResponse.body_as_unicode()`` is now deprecated, use :attr:`TextResponse.text ` instead (:issue:`4546`, :issue:`4555`, :issue:`4579`) @@ -2024,9 +5420,8 @@ New features * :ref:`Link extractors ` are now serializable, as long as you do not use :ref:`lambdas ` for parameters; for - example, you can now pass link extractors in :attr:`Request.cb_kwargs - ` or - :attr:`Request.meta ` when :ref:`persisting + example, you can now pass link extractors in :attr:`.Request.cb_kwargs` + or :attr:`.Request.meta` when :ref:`persisting scheduled requests ` (:issue:`4554`) * Upgraded the :ref:`pickle protocol ` that Scrapy uses @@ -2045,11 +5440,11 @@ Bug fixes * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer discards cookies defined in :attr:`Request.headers - ` (:issue:`1992`, :issue:`2400`) + ` (:issue:`1992`, :issue:`2400`) * :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter - of the ``__init__`` method of :class:`~scrapy.http.Request` + of the ``__init__`` method of :class:`~scrapy.Request` (:issue:`2400`, :issue:`3575`) * When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is @@ -2058,7 +5453,7 @@ Bug fixes * :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine syntax ` no longer need to return an iterable, and may - instead return a :class:`~scrapy.http.Request` object, an + instead return a :class:`~scrapy.Request` object, an :ref:`item `, or ``None`` (:issue:`4609`) * The :command:`startproject` command now ensures that the generated project @@ -2099,8 +5494,8 @@ Documentation :issue:`4587`) * The display-on-hover behavior of internal documentation references now also - covers links to :ref:`commands `, :attr:`Request.meta - ` keys, :ref:`settings ` and + covers links to :ref:`commands `, :attr:`.Request.meta` + keys, :ref:`settings ` and :ref:`signals ` (:issue:`4495`, :issue:`4563`) * It is again possible to download the documentation for offline reading @@ -2145,7 +5540,7 @@ Quality assurance * Added a `Pylint `_ job to Travis CI (:issue:`3727`) -* Added a `Mypy `_ job to Travis CI (:issue:`4637`) +* Added a `Mypy `_ job to Travis CI (:issue:`4637`) * Made use of set literals in tests (:issue:`4573`) @@ -2329,7 +5724,7 @@ Highlights: * :ref:`FTP support ` for media pipelines * New :attr:`Response.certificate ` attribute -* IPv6 support through :setting:`DNS_RESOLVER` +* IPv6 support through ``DNS_RESOLVER`` Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2385,7 +5780,7 @@ Deprecation removals ~~~~~~~~~~~~~~~~~~~~ * The :ref:`Scrapy shell ` no longer provides a `sel` proxy - object, use :meth:`response.selector ` + object, use :meth:`response.selector ` instead (:issue:`4347`) * LevelDB support has been removed (:issue:`4112`) @@ -2437,7 +5832,7 @@ New features :class:`twisted.internet.ssl.Certificate` object for HTTPS responses (:issue:`2726`, :issue:`4054`) -* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support +* A new ``DNS_RESOLVER`` setting allows enabling IPv6 support (:issue:`1031`, :issue:`4227`) * A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring @@ -2455,10 +5850,10 @@ New features * The new :attr:`Response.cb_kwargs ` attribute serves as a shortcut for :attr:`Response.request.cb_kwargs - ` (:issue:`4331`) + ` (:issue:`4331`) * :meth:`Response.follow ` now supports a - ``flags`` parameter, for consistency with :class:`~scrapy.http.Request` + ``flags`` parameter, for consistency with :class:`~scrapy.Request` (:issue:`4277`, :issue:`4279`) * :ref:`Item loader processors ` can now be @@ -2467,7 +5862,7 @@ New features * :class:`~scrapy.spiders.Rule` now accepts an ``errback`` parameter (:issue:`4000`) -* :class:`~scrapy.http.Request` no longer requires a ``callback`` parameter +* :class:`~scrapy.Request` no longer requires a ``callback`` parameter when an ``errback`` parameter is specified (:issue:`3586`, :issue:`4008`) * :class:`~scrapy.logformatter.LogFormatter` now supports some additional @@ -2497,7 +5892,7 @@ New features * :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError` exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls` - attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`, + attribute nor reimplement ``scrapy.spiders.Spider.start_requests()``, but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`) * :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use @@ -2517,7 +5912,7 @@ New features components already supported (:issue:`4126`) * :class:`scrapy.utils.python.MutableChain.__iter__` now returns ``self``, - `allowing it to be used as a sequence `_ + allowing it to be used as a sequence. (:issue:`4153`) @@ -2539,7 +5934,7 @@ Bug fixes * Redirects to URLs starting with 3 slashes (``///``) are now supported (:issue:`4032`, :issue:`4042`) -* :class:`~scrapy.http.Request` no longer accepts strings as ``url`` simply +* :class:`~scrapy.Request` no longer accepts strings as ``url`` simply because they have a colon (:issue:`2552`, :issue:`4094`) * The correct encoding is now used for attach names in @@ -2585,7 +5980,7 @@ Documentation using :class:`~scrapy.crawler.CrawlerProcess` (:issue:`2149`, :issue:`2352`, :issue:`3146`, :issue:`3960`) -* Clarified the requirements for :class:`~scrapy.http.Request` objects +* Clarified the requirements for :class:`~scrapy.Request` objects :ref:`when using persistence ` (:issue:`4124`, :issue:`4139`) @@ -2652,7 +6047,7 @@ Quality assurance * Cleaned up code (:issue:`3937`, :issue:`4208`, :issue:`4209`, :issue:`4210`, :issue:`4212`, :issue:`4369`, :issue:`4376`, :issue:`4378`) -.. _Bandit: https://bandit.readthedocs.io/ +.. _Bandit: https://bandit.readthedocs.io/en/latest/ .. _Flake8: https://flake8.pycqa.org/en/latest/ @@ -2681,7 +6076,7 @@ The following changes may impact custom priority queue classes: * A new keyword parameter has been added: ``key``. It is a string that is always an empty string for memory queues and indicates the - :setting:`JOB_DIR` value for disk queues. + :setting:`JOBDIR` value for disk queues. * The parameter for disk queues that contains data from the previous crawl, ``startprios`` or ``slot_startprios``, is now passed as a @@ -2761,6 +6156,38 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (2024-02-14) +------------------------- + +**Security bug fixes:** + +- Due to its `ReDoS vulnerabilities`_, ``scrapy.utils.iterators.xmliter`` is + now deprecated in favor of :func:`~scrapy.utils.iterators.xmliter_lxml`, + which :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace as a prefix in the node name, and big files with highly + nested trees when using libxml2 2.7+. + + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. + +- :setting:`DOWNLOAD_MAXSIZE` and :setting:`DOWNLOAD_WARNSIZE` now also apply + to the decompressed response body. Please, see the `7j7m-v7m3-jqm7 security + advisory`_ for more information. + +- Also in relation with the `7j7m-v7m3-jqm7 security advisory`_, use of the + ``scrapy.downloadermiddlewares.decompression`` module is discouraged and + will trigger a warning. + +- The ``Authorization`` header is now dropped on redirects to a different + domain. Please, see the `cw9j-q3vf-hrrv security advisory`_ for more + information. + + .. _cw9j-q3vf-hrrv security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cw9j-q3vf-hrrv + .. _release-1.8.3: @@ -2822,17 +6249,17 @@ Scrapy 1.8.2 (2022-03-01) **Security bug fixes:** -- When a :class:`~scrapy.http.Request` object with cookies defined gets a - redirect response causing a new :class:`~scrapy.http.Request` object to be +- When a :class:`~scrapy.Request` object with cookies defined gets a + redirect response causing a new :class:`~scrapy.Request` object to be scheduled, the cookies defined in the original - :class:`~scrapy.http.Request` object are no longer copied into the new - :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object are no longer copied into the new + :class:`~scrapy.Request` object. If you manually set the ``Cookie`` header on a - :class:`~scrapy.http.Request` object and the domain name of the redirect + :class:`~scrapy.Request` object and the domain name of the redirect URL is not an exact match for the domain of the URL of the original - :class:`~scrapy.http.Request` object, your ``Cookie`` header is now dropped - from the new :class:`~scrapy.http.Request` object. + :class:`~scrapy.Request` object, your ``Cookie`` header is now dropped + from the new :class:`~scrapy.Request` object. The old behavior could be exploited by an attacker to gain access to your cookies. Please, see the `cjvr-mfj7-j4j8 security advisory`_ for more @@ -2845,10 +6272,10 @@ Scrapy 1.8.2 (2022-03-01) ``example.com`` and any subdomain) by defining the shared domain suffix (e.g. ``example.com``) as the cookie domain when defining your cookies. See the documentation of the - :class:`~scrapy.http.Request` class for more information. + :class:`~scrapy.Request` class for more information. - When the domain of a cookie, either received in the ``Set-Cookie`` header - of a response or defined in a :class:`~scrapy.http.Request` object, is set + of a response or defined in a :class:`~scrapy.Request` object, is set to a `public suffix `_, the cookie is now ignored unless the cookie domain is the same as the request domain. @@ -2906,7 +6333,7 @@ Highlights: * Dropped Python 3.4 support and updated minimum requirements; made Python 3.8 support official -* New :meth:`Request.from_curl ` class method +* New :meth:`.Request.from_curl` class method * New :setting:`ROBOTSTXT_PARSER` and :setting:`ROBOTSTXT_USER_AGENT` settings * New :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` settings @@ -2914,6 +6341,8 @@ Highlights: Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. skip: start + * Python 3.4 is no longer supported, and some of the minimum requirements of Scrapy have also changed: @@ -2933,7 +6362,7 @@ Backward-incompatible changes consistency with similar classes (:issue:`3929`, :issue:`3982`) * If you are using a custom context factory - (:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must + (``DOWNLOADER_CLIENTCONTEXTFACTORY``), its ``__init__`` method must accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers`` (:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`) @@ -2954,20 +6383,22 @@ Backward-incompatible changes (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`, :issue:`4036`) +.. skip: end + See also :ref:`1.8-deprecation-removals` below. New features ~~~~~~~~~~~~ -* A new :meth:`Request.from_curl ` class +* A new :meth:`Request.from_curl ` class method allows :ref:`creating a request from a cURL command ` (:issue:`2985`, :issue:`3862`) * A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_ parser to use. It includes built-in support for :ref:`RobotFileParser `, - :ref:`Protego ` (default), :ref:`Reppy `, and + :ref:`Protego ` (default), Reppy, and :ref:`Robotexclusionrulesparser `, and allows you to :ref:`implement support for additional parsers ` (:issue:`754`, :issue:`2669`, @@ -2989,9 +6420,8 @@ New features ``True`` to enable debug-level messages about TLS connection parameters after establishing HTTPS connections (:issue:`2111`, :issue:`3450`) -* Callbacks that receive keyword arguments - (see :attr:`Request.cb_kwargs `) can now be - tested using the new :class:`@cb_kwargs +* Callbacks that receive keyword arguments (see :attr:`.Request.cb_kwargs`) + can now be tested using the new :class:`@cb_kwargs ` :ref:`spider contract ` (:issue:`3985`, :issue:`3988`) @@ -3167,8 +6597,8 @@ Backward-incompatible changes ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly. * :class:`~scrapy.crawler.Crawler`, - :class:`CrawlerRunner.crawl ` and - :class:`CrawlerRunner.create_crawler ` + :meth:`CrawlerRunner.crawl ` and + :meth:`CrawlerRunner.create_crawler ` no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they only accept a :class:`~scrapy.spiders.Spider` subclass now. @@ -3180,7 +6610,7 @@ Backward-incompatible changes * Non-default values for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting may stop working. Scheduler priority queue classes now need to handle - :class:`~scrapy.http.Request` objects instead of arbitrary Python data + :class:`~scrapy.Request` objects instead of arbitrary Python data structures. * An additional ``crawler`` parameter has been added to the ``__init__`` @@ -3200,9 +6630,9 @@ New features ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant scheduling improvement on crawls targeting multiple web domains, at the - cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) + cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`) -* A new :attr:`Request.cb_kwargs ` attribute +* A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods (:issue:`1138`, :issue:`3563`) @@ -3283,7 +6713,7 @@ Bug fixes * Requests with private callbacks are now correctly unserialized from disk (:issue:`3790`) -* :meth:`FormRequest.from_response() ` +* :meth:`.FormRequest.from_response` now handles invalid methods like major web browsers (:issue:`3777`, :issue:`3794`) @@ -3363,13 +6793,13 @@ The following deprecated APIs have been removed (:issue:`3578`): * From both ``scrapy.selector`` and ``scrapy.selector.lxmlsel``: - * ``HtmlXPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``HtmlXPathSelector`` (use :class:`~scrapy.Selector`) - * ``XmlXPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``XmlXPathSelector`` (use :class:`~scrapy.Selector`) - * ``XPathSelector`` (use :class:`~scrapy.selector.Selector`) + * ``XPathSelector`` (use :class:`~scrapy.Selector`) - * ``XPathSelectorList`` (use :class:`~scrapy.selector.Selector`) + * ``XPathSelectorList`` (use :class:`~scrapy.Selector`) * From ``scrapy.selector.csstranslator``: @@ -3379,7 +6809,7 @@ The following deprecated APIs have been removed (:issue:`3578`): * ``ScrapyXPathExpr`` (use parsel.csstranslator.XPathExpr_) -* From :class:`~scrapy.selector.Selector`: +* From :class:`~scrapy.Selector`: * ``_root`` (both the ``__init__`` method argument and the object property, use ``root``) @@ -3793,9 +7223,9 @@ Docs - Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting. (:issue:`2756`) - Update Contributing docs, document new support channels - (:issue:`2762`, issue:`3038`) + (:issue:`2762`, :issue:`3038`) - Include references to Scrapy subreddit in the docs -- Fix broken links; use https:// for external links +- Fix broken links; use ``https://`` for external links (:issue:`2978`, :issue:`2982`, :issue:`2958`) - Document CloseSpider extension better (:issue:`2759`) - Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example @@ -3909,7 +7339,7 @@ New Features (:issue:`2535`) - New :ref:`response.follow ` shortcut for creating requests (:issue:`1940`) -- Added ``flags`` argument and attribute to :class:`Request ` +- Added ``flags`` argument and attribute to :class:`~scrapy.Request` objects (:issue:`2047`) - Support Anonymous FTP (:issue:`2342`) - Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/`` @@ -3951,7 +7381,7 @@ Bug fixes - LinkExtractor now strips leading and trailing whitespaces from attributes (:issue:`2547`, fixes :issue:`1614`) - Properly handle whitespaces in action attribute in - :class:`~scrapy.http.FormRequest` (:issue:`2548`) + :class:`~scrapy.FormRequest` (:issue:`2548`) - Buffer CONNECT response bytes from proxy until all HTTP headers are received (:issue:`2495`, fixes :issue:`2491`) - FTP downloader now works on Python 3, provided you use Twisted>=17.1 @@ -3993,8 +7423,7 @@ Documentation ~~~~~~~~~~~~~ - Binary mode is required for exporters (:issue:`2564`, fixes :issue:`2553`) -- Mention issue with :meth:`FormRequest.from_response - ` due to bug in lxml (:issue:`2572`) +- Mention issue with :meth:`.FormRequest.from_response` due to bug in lxml (:issue:`2572`) - Use single quotes uniformly in templates (:issue:`2596`) - Document :reqmeta:`ftp_user` and :reqmeta:`ftp_password` meta keys (:issue:`2587`) - Removed section on deprecated ``contrib/`` (:issue:`2636`) @@ -4389,14 +7818,14 @@ This 1.1 release brings a lot of interesting features and bug fixes: selectors engine without needing to upgrade Scrapy. - HTTPS downloader now does TLS protocol negotiation by default, instead of forcing TLS 1.0. You can also set the SSL/TLS method - using the new :setting:`DOWNLOADER_CLIENT_TLS_METHOD`. + using the new ``DOWNLOADER_CLIENT_TLS_METHOD`` setting. - These bug fixes may require your attention: - Don't retry bad requests (HTTP 400) by default (:issue:`1289`). If you need the old behavior, add ``400`` to :setting:`RETRY_HTTP_CODES`. - Fix shell files argument handling (:issue:`1710`, :issue:`1550`). - If you try ``scrapy shell index.html`` it will try to load the URL http://index.html, + If you try ``scrapy shell index.html`` it will try to load the URL ``http://index.html``, use ``scrapy shell ./index.html`` to load a local file. - Robots.txt compliance is now enabled by default for newly-created projects (:issue:`1724`). Scrapy will also wait for robots.txt to be downloaded @@ -4424,8 +7853,7 @@ Keep reading for more details on other improvements and bug fixes. Beta Python 3 Support ~~~~~~~~~~~~~~~~~~~~~ -We have been `hard at work to make Scrapy run on Python 3 -`_. As a result, now +We have been hard at work to make Scrapy run on Python 3. As a result, now you can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some features are still missing (and some may never be ported). @@ -4493,7 +7921,7 @@ Additional New Features and Enhancements - Other refactoring, optimizations and cleanup (:issue:`1476`, :issue:`1481`, :issue:`1477`, :issue:`1315`, :issue:`1290`, :issue:`1750`, :issue:`1881`). -.. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md +.. _Code of Conduct: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md Deprecations and Removals @@ -4533,7 +7961,7 @@ Bugfixes - Support empty password for http_proxy config (:issue:`1274`). - Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`). - Support link rel attribute with multiple values (:issue:`1201`). -- Fixed ``scrapy.http.FormRequest.from_response`` when there is a ```` +- Fixed ``scrapy.FormRequest.from_response`` when there is a ```` tag (:issue:`1564`). - Fixed :setting:`TEMPLATES_DIR` handling (:issue:`1575`). - Various ``FormRequest`` fixes (:issue:`1595`, :issue:`1596`, :issue:`1597`). @@ -5072,7 +8500,7 @@ Scrapy 0.24.5 (2015-02-25) Scrapy 0.24.4 (2014-08-09) -------------------------- -- pem file is used by mockserver and required by scrapy bench (:commit:`5eddc68`) +- pem file is used by mockserver and required by scrapy bench (:commit:`5eddc68b63`) - scrapy bench needs scrapy.tests* (:commit:`d6cb999`) Scrapy 0.24.3 (2014-08-09) @@ -5255,7 +8683,7 @@ Enhancements - Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`) - Improve test coverage and forthcoming Python 3 support (:issue:`525`) - Promote startup info on settings and middleware to INFO level (:issue:`520`) -- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`) +- Support partials in ``get_func_args`` util (:issue:`506`, :issue:`504`) - Allow running individual tests via tox (:issue:`503`) - Update extensions ignored by link extractors (:issue:`498`) - Add middleware methods to get files/images/thumbs paths (:issue:`490`) @@ -5388,7 +8816,7 @@ Scrapy 0.18.4 (released 2013-10-10) - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`) - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`) -- Fix start_requests laziness and early hangs (:commit:`89faf52`) +- Fix ``start_requests()`` laziness and early hangs (:commit:`89faf52`) Scrapy 0.18.3 (released 2013-10-03) ----------------------------------- @@ -5460,7 +8888,7 @@ Scrapy 0.18.0 (released 2013-08-09) - Moved persistent (on disk) queues to a separate project (queuelib_) which Scrapy now depends on - Add Scrapy commands using external libraries (:issue:`260`) - Added ``--pdb`` option to ``scrapy`` command line tool -- Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. +- Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. - Several improvements to spider contracts - New default middleware named MetaRefreshMiddleware that handles meta-refresh html tag redirections, - MetaRefreshMiddleware and RedirectMiddleware have different priorities to address #62 @@ -5581,7 +9009,7 @@ Scrapy changes: - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. -- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares +- added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` @@ -5593,7 +9021,7 @@ Scrapy changes: - nested items now fully supported in JSON and JSONLines exporters - added :reqmeta:`cookiejar` Request meta key to support multiple cookie sessions per spider - decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that module -- dropped support for Python 2.5. See https://blog.scrapinghub.com/2012/02/27/scrapy-0-15-dropping-support-for-python-2-5/ +- dropped support for Python 2.5. See https://www.zyte.com/blog/scrapy-0-15-dropping-support-for-python-2-5/ - dropped support for Twisted 2.5 - added :setting:`REFERER_ENABLED` setting, to control referer middleware - changed default user agent to: ``Scrapy/VERSION (+http://scrapy.org)`` @@ -5644,7 +9072,7 @@ Scrapy 0.14.2 - fixed bug in MemoryUsage extension: get_engine_status() takes exactly 1 argument (0 given) (:commit:`11133e9`) - fixed struct.error on http compression middleware. closes #87 (:commit:`1423140`) - ajax crawling wasn't expanding for unicode urls (:commit:`0de3fb4`) -- Catch start_requests iterator errors. refs #83 (:commit:`454a21d`) +- Catch ``start_requests()`` iterator errors. refs #83 (:commit:`454a21d`) - Speed-up libxml2 XPathSelector (:commit:`2fbd662`) - updated versioning doc according to recent changes (:commit:`0a070f5`) - scrapyd: fixed documentation link (:commit:`2b4e4c3`) @@ -5671,7 +9099,7 @@ Scrapy 0.14 New features and settings ~~~~~~~~~~~~~~~~~~~~~~~~~ -- Support for `AJAX crawlable urls`_ +- Support for AJAX crawlable urls - New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`) - added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``) - Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`) @@ -5681,7 +9109,7 @@ New features and settings - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` + - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP`` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) @@ -5823,7 +9251,7 @@ API changes - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` - ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler`` - Moved module: ``scrapy.contrib.spidermanager`` to ``scrapy.spidermanager`` -- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders` attribute of ``scrapy.project.crawler`` singleton. +- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders`` attribute of ``scrapy.project.crawler`` singleton. - moved Stats Collector classes: (#204) - ``scrapy.stats.collector.StatsCollector`` to ``scrapy.statscol.StatsCollector`` - ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector`` @@ -5942,11 +9370,10 @@ Scrapy 0.7 First release of Scrapy. -.. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 .. _boto3: https://github.com/boto/boto3 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding -.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ +.. _ClientForm: https://pypi.org/project/ClientForm/ .. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request .. _cryptography: https://cryptography.io/en/latest/ .. _docstrings: https://docs.python.org/3/glossary.html#term-docstring @@ -5958,8 +9385,8 @@ First release of Scrapy. .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr -.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ -.. _Pillow: https://python-pillow.org/ +.. _PEP 257: https://peps.python.org/pep-0257/ +.. _Pillow: https://github.com/python-pillow/Pillow .. _pyOpenSSL: https://www.pyopenssl.org/en/stable/ .. _queuelib: https://github.com/scrapy/queuelib .. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml @@ -5970,7 +9397,7 @@ First release of Scrapy. .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ -.. _Twisted: https://twistedmatrix.com/trac/ +.. _Twisted: https://twisted.org/ .. _w3lib: https://github.com/scrapy/w3lib .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 000000000..3783dd1dc --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,8 @@ +h2 +pydantic +scrapy-spider-metadata +sphinx +sphinx-notfound-page +sphinx-rtd-theme +sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10 diff --git a/docs/requirements.txt b/docs/requirements.txt index 9f9aef711..0f5969401 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,197 @@ -sphinx==5.0.2 -sphinx-hoverxref==1.1.1 -sphinx-notfound-page==0.8 -sphinx-rtd-theme==1.0.0 +# This file was autogenerated by uv via the following command: +# uv pip compile -p 3.13 docs/requirements.in -o docs/requirements.txt +alabaster==1.0.0 + # via sphinx +annotated-types==0.7.0 + # via pydantic +attrs==26.1.0 + # via + # service-identity + # twisted +automat==25.4.16 + # via twisted +babel==2.18.0 + # via sphinx +certifi==2026.2.25 + # via requests +cffi==2.0.0 + # via cryptography +charset-normalizer==3.4.6 + # via requests +constantly==23.10.4 + # via twisted +cryptography==46.0.6 + # via + # pyopenssl + # scrapy + # service-identity +cssselect==1.4.0 + # via + # parsel + # scrapy +defusedxml==0.7.1 + # via scrapy +docutils==0.22.4 + # via + # sphinx + # sphinx-markdown-builder + # sphinx-rtd-theme +filelock==3.25.2 + # via tldextract +h2==4.3.0 + # via -r docs/requirements.in +hpack==4.1.0 + # via h2 +hyperframe==6.1.0 + # via h2 +hyperlink==21.0.0 + # via twisted +idna==3.11 + # via + # hyperlink + # requests + # tldextract +imagesize==2.0.0 + # via sphinx +incremental==24.11.0 + # via twisted +itemadapter==0.13.1 + # via + # itemloaders + # scrapy +itemloaders==1.4.0 + # via scrapy +jinja2==3.1.6 + # via sphinx +jmespath==1.1.0 + # via + # itemloaders + # parsel +lxml==6.0.2 + # via + # parsel + # scrapy +markupsafe==3.0.3 + # via jinja2 +packaging==26.0 + # via + # incremental + # parsel + # scrapy + # scrapy-spider-metadata + # sphinx + # sphinx-scrapy +parsel==1.11.0 + # via + # itemloaders + # scrapy +protego==0.6.0 + # via scrapy +pyasn1==0.6.3 + # via + # pyasn1-modules + # service-identity +pyasn1-modules==0.4.2 + # via service-identity +pycparser==3.0 + # via cffi +pydantic==2.12.5 + # via + # -r docs/requirements.in + # scrapy-spider-metadata +pydantic-core==2.41.5 + # via pydantic +pydispatcher==2.0.7 + # via scrapy +pygments==2.19.2 + # via sphinx +pyopenssl==26.0.0 + # via scrapy +queuelib==1.9.0 + # via scrapy +requests==2.33.0 + # via + # requests-file + # sphinx + # tldextract +requests-file==3.0.1 + # via tldextract +roman-numerals==4.1.0 + # via sphinx +scrapy==2.14.2 + # via scrapy-spider-metadata +scrapy-spider-metadata==0.2.0 + # via -r docs/requirements.in +service-identity==24.2.0 + # via scrapy +snowballstemmer==3.0.1 + # via sphinx +sphinx==9.1.0 + # via + # -r docs/requirements.in + # sphinx-copybutton + # sphinx-last-updated-by-git + # sphinx-llms-txt + # sphinx-markdown-builder + # sphinx-notfound-page + # sphinx-rtd-theme + # sphinx-scrapy + # sphinxcontrib-jquery +sphinx-copybutton==0.5.2 + # via sphinx-scrapy +sphinx-last-updated-by-git==0.3.8 + # via sphinx-sitemap +sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69 + # via sphinx-scrapy +sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@cfe4c0bfd7b4542f7e6b65a58cdf9ec765829940 + # via sphinx-scrapy +sphinx-notfound-page==1.1.0 + # via -r docs/requirements.in +sphinx-rtd-dark-mode==1.3.0 + # via -r docs/requirements.in +sphinx-rtd-theme==3.1.0 + # via + # -r docs/requirements.in + # sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8 + # via -r docs/requirements.in +sphinx-sitemap==2.9.0 + # via sphinx-scrapy +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +tabulate==0.10.0 + # via sphinx-markdown-builder +tldextract==5.3.1 + # via scrapy +twisted==25.5.0 + # via scrapy +typing-extensions==4.15.0 + # via + # pydantic + # pydantic-core + # twisted + # typing-inspection +typing-inspection==0.4.2 + # via pydantic +urllib3==2.6.3 + # via requests +w3lib==2.4.1 + # via + # parsel + # scrapy +zope-interface==8.2 + # via + # scrapy + # twisted diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 1bf2172bd..75f15b3ae 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -21,10 +21,14 @@ The ``ADDONS`` setting is a dict in which every key is an add-on class or its import path and the value is its priority. This is an example where two add-ons are enabled in a project's -``settings.py``:: +``settings.py``: + +.. skip: next + +.. code-block:: python ADDONS = { - 'path.to.someaddon': 0, + "path.to.someaddon": 0, SomeAddonClass: 1, } @@ -32,7 +36,8 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are Python classes that include the following method: +Add-ons are :ref:`components ` that include one or both of +the following methods: .. method:: update_settings(settings) @@ -45,37 +50,30 @@ Add-ons are Python classes that include the following method: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` -They can also have the following method: +.. classmethod:: update_pre_crawler_settings(cls, settings) -.. classmethod:: from_crawler(cls, crawler) - :noindex: + Use this class method instead of the :meth:`update_settings` method to + update :ref:`pre-crawler settings ` whose value is + used before the :class:`~scrapy.crawler.Crawler` object is created. - If present, this class method is called to create an add-on instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the add-on. The crawler object provides access to all Scrapy core - components like settings and signals; it is a way for the add-on to access - them and hook its functionality into Scrapy. - - :param crawler: The crawler that uses this add-on - :type crawler: :class:`~scrapy.crawler.Crawler` + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.BaseSettings` The settings set by the add-on should use the ``addon`` priority (see -:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`): + +.. code-block:: python class MyAddon: def update_settings(self, settings): settings.set("DNSCACHE_ENABLED", True, "addon") This allows users to override these settings in the project or spider -configuration. This is not possible with settings that are mutable objects, -such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases -you can provide an add-on-specific setting that governs whether the add-on will -modify :setting:`ITEM_PIPELINES`:: +configuration. - class MyAddon: - def update_settings(self, settings): - if settings.getbool("MYADDON_ENABLE_PIPELINE"): - settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 +When editing the value of a setting instead of overriding it entirely, it is +usually best to leave its priority unchanged. For example, when editing a +:ref:`component priority dictionary `. If the ``update_settings`` method raises :exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes @@ -96,7 +94,7 @@ recommend that such custom components should be written in the following way: 1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the default Scrapy one (e.g. - ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead + ``scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler``), but instead be able to load the class of the fallback component from a special setting (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use it. @@ -106,9 +104,9 @@ recommend that such custom components should be written in the following way: (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default setting to the component provided by the add-on (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, - they shouldn't change it. + it should not be changed. 3. This way, if there are several add-ons that want to modify the same setting, - all of them will fallback to the component from the previous one and then to + all of them will fall back to the component from the previous one and then to the Scrapy default. The order of that depends on the priority order in the ``ADDONS`` setting. @@ -118,12 +116,30 @@ Add-on examples Set some basic configuration: +.. skip: next .. code-block:: python + from myproject.pipelines import MyPipeline + + class MyAddon: def update_settings(self, settings): - settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings.set("DNSCACHE_ENABLED", True, "addon") + settings.remove_from_list("METAREFRESH_IGNORE_TAGS", "noscript") + settings.setdefault_in_component_priority_dict( + "ITEM_PIPELINES", MyPipeline, 200 + ) + +.. _priority-dict-helpers: + +.. tip:: When editing a :ref:`component priority dictionary + ` setting, like :setting:`ITEM_PIPELINES`, + consider using setting methods like + :meth:`~scrapy.settings.BaseSettings.replace_in_component_priority_dict`, + :meth:`~scrapy.settings.BaseSettings.set_in_component_priority_dict` + and + :meth:`~scrapy.settings.BaseSettings.setdefault_in_component_priority_dict` + to avoid mistakes. Check dependencies: @@ -150,15 +166,13 @@ Access the crawler instance: def from_crawler(cls, crawler): return cls(crawler) - def update_settings(self, settings): - ... + def update_settings(self, settings): ... Use a fallback component: .. code-block:: python - from scrapy.core.downloader.handlers.http import HTTPDownloadHandler - + from scrapy.utils.misc import build_from_crawler, load_object FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -166,20 +180,19 @@ Use a fallback component: class MyHandler: lazy = False - def __init__(self, settings, crawler): - dhcls = load_object(settings.get(FALLBACK_SETTING)) - self._fallback_handler = create_instance( - dhcls, - settings=None, - crawler=crawler, - ) + def __init__(self, crawler): + dhcls = load_object(crawler.settings.get(FALLBACK_SETTING)) + self._fallback_handler = build_from_crawler(dhcls, crawler) - def download_request(self, request, spider): + async def download_request(self, request): if request.meta.get("my_params"): # handle the request ... else: - return self._fallback_handler.download_request(request, spider) + return await self._fallback_handler.download_request(request) + + async def close(self): + pass class MyAddon: diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 175c877de..7ff8d3464 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -12,10 +12,11 @@ extensions and middlewares. Crawler API =========== -The main entry point to Scrapy API is the :class:`~scrapy.crawler.Crawler` -object, passed to extensions through the ``from_crawler`` class method. This -object provides access to all Scrapy core components, and it's the only way for -extensions to access them and hook their functionality into Scrapy. +The main entry point to the Scrapy API is the :class:`~scrapy.crawler.Crawler` +object, which :ref:`components ` can :ref:`get for +initialization `. It provides access to all Scrapy core +components, and it is the only way for components to access them and hook their +functionality into Scrapy. .. module:: scrapy.crawler :synopsis: The Scrapy crawler @@ -26,7 +27,9 @@ contains a dictionary of all available extensions and their order similar to how you :ref:`configure the downloader middlewares `. -.. class:: Crawler(spidercls, settings) +.. autoclass:: Crawler + :members: get_addon, get_downloader_middleware, get_extension, + get_item_pipeline, get_spider_middleware The Crawler object must be instantiated with a :class:`scrapy.Spider` subclass and a @@ -86,7 +89,7 @@ how you :ref:`configure the downloader middlewares The execution engine, which coordinates the core crawling logic between the scheduler, downloader and spiders. - Some extension may want to access the Scrapy engine, to inspect or + Some extension may want to access the Scrapy engine, to inspect or modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. @@ -96,19 +99,25 @@ how you :ref:`configure the downloader middlewares provided while constructing the crawler, and it is created after the arguments given in the :meth:`crawl` method. - .. method:: crawl(*args, **kwargs) + .. automethod:: crawl_async - Starts the crawler by instantiating its spider class with the given - ``args`` and ``kwargs`` arguments, while setting the execution engine in - motion. Should be called only once. + .. automethod:: crawl - Returns a deferred that is fired when the crawl is finished. + .. automethod:: stop_async .. automethod:: stop +.. autoclass:: AsyncCrawlerRunner + :members: + .. autoclass:: CrawlerRunner :members: +.. autoclass:: AsyncCrawlerProcess + :show-inheritance: + :members: + :inherited-members: + .. autoclass:: CrawlerProcess :show-inheritance: :members: @@ -163,46 +172,17 @@ SpiderLoader API .. module:: scrapy.spiderloader :synopsis: The spider loader -.. class:: SpiderLoader +Custom spider loaders can be employed by specifying their path in the +:setting:`SPIDER_LOADER_CLASS` project setting. They must implement +:class:`SpiderLoaderProtocol`. - This class is in charge of retrieving and handling the spider classes - defined across the project. +.. autoclass:: SpiderLoaderProtocol + :members: - Custom spider loaders can be employed by specifying their path in the - :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement - the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an - errorless execution. +.. autoclass:: SpiderLoader + :members: - .. method:: from_settings(settings) - - This class method is used by Scrapy to create an instance of the class. - It's called with the current project settings, and it loads the spiders - found recursively in the modules of the :setting:`SPIDER_MODULES` - setting. - - :param settings: project settings - :type settings: :class:`~scrapy.settings.Settings` instance - - .. method:: load(spider_name) - - Get the Spider class with the given name. It'll look into the previously - loaded spiders for a spider class with name ``spider_name`` and will raise - a KeyError if not found. - - :param spider_name: spider class name - :type spider_name: str - - .. method:: list() - - Get the names of the available spiders in the project. - - .. method:: find_by_request(request) - - List the spiders' names that can handle the given request. Will try to - match the request's url against the domains of the spiders. - - :param request: queried request - :type request: :class:`~scrapy.Request` instance +.. autoclass:: DummySpiderLoader .. _topics-api-signals: @@ -269,11 +249,17 @@ class (which they all inherit from). The following methods are not part of the stats collection api but instead used when implementing custom stats collectors: - .. method:: open_spider(spider) + .. method:: open_spider() - Open the given spider for stats collection. + Open the spider for stats collection. - .. method:: close_spider(spider) + .. method:: close_spider() - Close the given spider. After this is called, no more specific stats + Close the spider. After this is called, no more specific stats can be accessed or collected. + +Engine API +========== + +.. autoclass:: scrapy.core.engine.ExecutionEngine() + :members: needs_backout diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 0c3a7ed88..c60c43f3c 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -63,7 +63,7 @@ this: :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`). 8. The :ref:`Engine ` sends processed items to - :ref:`Item Pipelines `, then send processed Requests to + :ref:`Item Pipelines `, then sends processed Requests to the :ref:`Scheduler ` and asks for possible next Requests to crawl. @@ -87,8 +87,8 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The :ref:`scheduler ` receives requests from the engine and -enqueues them for feeding them later (also to the engine) when the engine +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine requests them. .. _component-downloader: @@ -150,7 +150,7 @@ requests). Use a Spider middleware if you need to * post-process output of spider callbacks - change/add/remove requests or items; -* post-process start_requests; +* post-process start requests or items; * handle spider exceptions; * call errback instead of callback for some of the requests based on response content. @@ -168,9 +168,7 @@ For more information about asynchronous programming and Twisted see these links: * :doc:`twisted:core/howto/defer-intro` -* `Twisted - hello, asynchronous programming`_ * `Twisted Introduction - Krondo`_ -.. _Twisted: https://twistedmatrix.com/trac/ -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ -.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ +.. _Twisted: https://twisted.org/ +.. _Twisted Introduction - Krondo: https://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 07baea071..afccb491d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -4,27 +4,38 @@ asyncio ======= -.. versionadded:: 2.0 +Scrapy supports :mod:`asyncio` natively. New projects created with +:command:`startproject` have asyncio enabled by default, and you can use +:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine +`. -Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the -asyncio reactor `, you may use :mod:`asyncio` and -:mod:`asyncio`-powered libraries in any :doc:`coroutine `. +The rest of this page covers advanced topics. If you are starting a new project, +no additional setup is needed. .. _install-asyncio: -Installing the asyncio reactor -============================== +Configuring the asyncio reactor +=============================== -To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to -``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``. +New projects generated with :command:`startproject` have the asyncio +reactor configured by default. No manual setup is needed. -If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to +The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy +uses. Its default value is +``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which enables +:mod:`asyncio` support. + +If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.CrawlerRunner`, you also need to install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` reactor manually. You can do that using -:func:`~scrapy.utils.reactor.install_reactor`:: +:func:`~scrapy.utils.reactor.install_reactor`: - install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') +.. skip: next +.. code-block:: python + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") .. _asyncio-preinstalled-reactor: @@ -44,6 +55,7 @@ You can usually fix the issue by moving those offending module-level Twisted imports to the method or function definitions where they are used. For example, if you have something like: +.. skip: next .. code-block:: python from twisted.internet import reactor @@ -68,24 +80,36 @@ those imports happen. .. _asyncio-await-dfd: -Awaiting on Deferreds -===================== +Integrating Deferred code and asyncio code +========================================== -When the asyncio reactor isn't installed, you can await on Deferreds in the -coroutines directly. When it is installed, this is not possible anymore, due to -specifics of the Scrapy coroutine integration (the coroutines are wrapped into -:class:`asyncio.Future` objects, not into -:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into -Futures. Scrapy provides two helpers for this: +Coroutine functions can await on Deferreds by wrapping them into +:class:`asyncio.Future` objects. Scrapy provides two helpers for this: .. autofunction:: scrapy.utils.defer.deferred_to_future .. autofunction:: scrapy.utils.defer.maybe_deferred_to_future + +.. tip:: If you don't need to support reactors other than the default + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, you + can use :func:`~scrapy.utils.defer.deferred_to_future`, otherwise you + should use :func:`~scrapy.utils.defer.maybe_deferred_to_future`. + .. tip:: If you need to use these functions in code that aims to be compatible with lower versions of Scrapy that do not provide these functions, down to Scrapy 2.0 (earlier versions do not support :mod:`asyncio`), you can copy the implementation of these functions into your own code. +Coroutines and futures can be wrapped into Deferreds (for example, when a +Scrapy API requires passing a Deferred to it) using the following helpers: + +.. autofunction:: scrapy.utils.defer.deferred_from_coro +.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f + +The following function helps with a reverse wrapping: + +.. autofunction:: scrapy.utils.defer.ensure_awaitable + .. _enforce-asyncio-requirement: @@ -93,25 +117,203 @@ Enforcing asyncio as a requirement ================================== If you are writing a :ref:`component ` that requires asyncio -to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to +to work, use :func:`scrapy.utils.asyncio.is_asyncio_available` to :ref:`enforce it as a requirement `. For example: .. code-block:: python - from scrapy.utils.reactor import is_asyncio_reactor_installed + from scrapy.utils.asyncio import is_asyncio_available class MyComponent: def __init__(self): - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): raise ValueError( - f"{MyComponent.__qualname__} requires the asyncio Twisted " - f"reactor. Make sure you have it configured in the " + f"{MyComponent.__qualname__} requires the asyncio support. " + f"Make sure you have configured the asyncio reactor in the " f"TWISTED_REACTOR setting. See the asyncio documentation " f"of Scrapy for more information." ) +.. autofunction:: scrapy.utils.asyncio.is_asyncio_available +.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed + + +.. _asyncio-without-reactor: + +Using Scrapy without a Twisted reactor +====================================== + +.. versionadded:: 2.15.0 + +.. warning:: + This is currently experimental and may not be suitable for production use. + +.. note:: As the Twisted download handlers cannot be used without a reactor, + the default download handler in this mode is + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You + will need to additionally install the :ref:`httpx ` extra to use + it, unless you switch to some different handler. + +It's possible to use Scrapy without installing a Twisted reactor at all, by +setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this +mode Scrapy will use the asyncio event loop directly, and most of the Scrapy +functionality will work in the same way. + +Doing this provides several benefits in certain use cases: + +* A Twisted reactor, once stopped, cannot be started again. This prevents, for + example, using several instances of + :class:`~scrapy.crawler.AsyncCrawlerProcess` in the same process when they + use a reactor, but with ``TWISTED_REACTOR_ENABLED=False`` it becomes + possible. +* There may be limitations imposed by + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` and related + Twisted code, such as the requirement of using + :class:`~asyncio.SelectorEventLoop` on Windows (see :ref:`asyncio-windows`), + that do not apply if the reactor is not used. +* :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` manages the + underlying event loop, and while :class:`~scrapy.crawler.AsyncCrawlerRunner` + can use a pre-existing reactor which, in turn, can use a pre-existing event + loop, it's easier to use :class:`~scrapy.crawler.AsyncCrawlerRunner` with a + pre-existing loop directly. +* Omitting the reactor machinery may improve performance and reliability. + +Limitations +----------- + +As some Scrapy features and components require a reactor, they don't work and +are disabled without it. Replacements that don't require a reactor may be added +in future Scrapy versions. The following features are not available: + +* The default HTTP(S) download handler, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` (this + is likely the biggest difference; Scrapy provides an HTTP(S) download handler + that doesn't require a reactor and will be used instead of it: + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`) +* :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` +* :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` +* :ref:`topics-telnetconsole` +* :class:`~scrapy.crawler.CrawlerRunner` and + :class:`~scrapy.crawler.CrawlerProcess` + (:class:`~scrapy.crawler.AsyncCrawlerProcess` and + :class:`~scrapy.crawler.AsyncCrawlerRunner` are available) +* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting) +* User and 3rd-party code that requires a reactor (see :ref:`below + ` for examples) + +Note that importing Twisted modules and, among other things, creating and using +:class:`~twisted.internet.defer.Deferred` objects doesn't require a reactor, so +code that uses :class:`~twisted.internet.defer.Deferred`, +:class:`~twisted.python.failure.Failure` and some other Twisted APIs will not +necessarily stop working. + +Other differences +----------------- + +When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Scrapy will change +the defaults of some other settings: + +* :setting:`TELNETCONSOLE_ENABLED` is set to ``False``. +* The ``"http"`` and ``"https"`` keys in :setting:`DOWNLOAD_HANDLERS_BASE` are + set to ``"scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler"``. +* The ``"ftp"`` key in :setting:`DOWNLOAD_HANDLERS_BASE` is set to ``None``. + +Thus, :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` is +used by default for making HTTP(S) requests. Please refer to its documentation +for its differences and limitations compared to +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + +Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a +:term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from +being imported. It will be uninstalled when :meth:`AsyncCrawlerProcess.start() +` exits. + +.. _asyncio-without-reactor-migrate: + +Adding support to existing code +------------------------------- + +Code that doesn't directly use Twisted APIs or APIs that depend on Twisted ones +doesn't need special support for running without a reactor. + +Here are some examples of APIs and patterns that need a replacement: + +* Using :meth:`reactor.callLater() + ` for sleeping or delayed calls. + You can use :meth:`asyncio.loop.call_later` instead. +* Using :func:`twisted.internet.threads.deferToThread`, + :meth:`reactor.callFromThread() + ` and related APIs to + execute code in other threads. You can use :func:`asyncio.to_thread`, + :meth:`asyncio.loop.call_soon_threadsafe` and related APIs instead. +* Using :class:`twisted.internet.task.LoopingCall` for scheduling repeated + tasks. As there is no direct replacement in the standard library, you may + need to write your own one using :func:`asyncio.sleep` in a task. +* Using Twisted network client and server APIs (:meth:`reactor.connectTCP() + `, + :meth:`reactor.listenTCP() + `, + :mod:`twisted.web.client`, :mod:`twisted.mail.smtp` etc.). You can use other + built-in or 3rd-party libraries for this. +* Using :class:`~scrapy.crawler.CrawlerProcess` or + :class:`~scrapy.crawler.CrawlerRunner`. You should use + :class:`~scrapy.crawler.AsyncCrawlerProcess` or + :class:`~scrapy.crawler.AsyncCrawlerRunner` respectively instead. +* Checking whether ``asyncio`` support is available with + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`. You should use + :func:`scrapy.utils.asyncio.is_asyncio_available` instead. + +Scrapy provides unified helpers for some of these examples: + +.. autofunction:: scrapy.utils.asyncio.sleep +.. autofunction:: scrapy.utils.asyncio.call_later +.. autofunction:: scrapy.utils.asyncio.create_looping_call +.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall +.. autofunction:: scrapy.utils.asyncio.run_in_thread + +If your code needs to know whether the reactor is available, you can either +check for the value of the :setting:`TWISTED_REACTOR_ENABLED` setting (you need +access to the :class:`~scrapy.crawler.Crawler` instance to do this) or use the +following function: + +.. autofunction:: scrapy.utils.reactorless.is_reactorless + +In general, code that doesn't use the reactor (directly or indirectly) can be +used unmodified both with the asyncio reactor and without a reactor. This +includes code that converts Deferreds to futures and vice versa as described in +:ref:`asyncio-await-dfd`. + +Troubleshooting +--------------- + +**ImportError: Import of twisted.internet.reactor is forbidden when running +without a Twisted reactor [...]:** Scrapy is configured to run without a +reactor, but some code imported :mod:`twisted.internet.reactor`, most likely +because that code needs a reactor to be used. You need to stop using this code +or set :setting:`TWISTED_REACTOR_ENABLED` back to ``True``. It's also possible +that the reactor isn't really needed but was installed due to the problem +described in :ref:`asyncio-preinstalled-reactor`, in which case it should be +enough to fix the problematic imports. + +**RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is +installed:** Scrapy is configured to run without a reactor, but a reactor is +already installed before the Scrapy code is executed. If you are trying to set +:setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. + +**RuntimeError: We expected a Twisted reactor to be installed but it isn't:** +Scrapy is configured to run with a reactor and not to install one, but a +reactor wasn't installed before the Scrapy code is executed. If you are trying +to set :setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. + +**RuntimeError: doesn't support TWISTED_REACTOR_ENABLED=False:** The +listed class cannot be used with :setting:`TWISTED_REACTOR_ENABLED` set to +``False``. There may be a replacement in the :ref:`documentation above +` or the documentation of the affected class. + .. _asyncio-windows: @@ -124,8 +326,7 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and :class:`~asyncio.SelectorEventLoop` works with Twisted. Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop` -automatically when you change the :setting:`TWISTED_REACTOR` setting or call -:func:`~scrapy.utils.reactor.install_reactor`. +automatically when installing the asyncio reactor. .. note:: Other libraries you use may require :class:`~asyncio.ProactorEventLoop`, e.g. because it supports @@ -133,6 +334,9 @@ automatically when you change the :setting:`TWISTED_REACTOR` setting or call them together with Scrapy on Windows (but you should be able to use them on WSL or native Linux). +.. note:: This problem doesn't apply when not using the reactor, see + :ref:`asyncio-without-reactor`. + .. _playwright: https://github.com/microsoft/playwright-python @@ -144,3 +348,18 @@ Using custom asyncio loops You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. + + +.. _disable-asyncio: + +Switching to a non-asyncio reactor +================================== + +If for some reason your code doesn't work with the asyncio reactor, you can use +a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its +import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to +``None``, which will use the default reactor for your platform. If you are +using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.AsyncCrawlerProcess` you also need to switch to their +Deferred-based counterparts: :class:`~scrapy.crawler.CrawlerRunner` or +:class:`~scrapy.crawler.CrawlerProcess` respectively. diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 8e6aae65c..33289545c 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -21,9 +21,14 @@ Design goals How it works ============ -AutoThrottle extension adjusts download delays dynamically to make spider send -:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent requests on average -to each remote website. +Scrapy allows defining the concurrency and delay of different download slots, +e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are +assigned to slots based on their URL domain, although it is possible to +customize the download slot of any request. + +The AutoThrottle extension adjusts the delay of each download slot dynamically, +to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent +requests on average to each remote website. It uses download latency to compute the delays. The main idea is the following: if a server needs ``latency`` seconds to respond, a client @@ -32,8 +37,7 @@ processed in parallel. Instead of adjusting the delays one can just set a small fixed download delay and impose hard limits on concurrency using -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. It will provide a similar effect, but there are some important differences: * because the download delay is small there will be occasional bursts @@ -66,13 +70,12 @@ AutoThrottle algorithm adjusts download delays based on the following rules: .. note:: The AutoThrottle extension honours the standard Scrapy settings for concurrency and delay. This means that it will respect :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`CONCURRENT_REQUESTS_PER_IP` options and never set a download delay lower than :setting:`DOWNLOAD_DELAY`. .. _download-latency: In Scrapy, the download latency is measured as the time elapsed between -establishing the TCP connection and receiving the HTTP headers. +sending the request and receiving the HTTP headers. Note that these latencies are very hard to measure accurately in a cooperative multitasking environment because Scrapy may be busy processing a spider @@ -80,6 +83,34 @@ callback, for example, and unable to attend downloads. However, these latencies should still give a reasonable estimate of how busy Scrapy (and ultimately, the server) is, and this extension builds on that premise. +.. reqmeta:: autothrottle_dont_adjust_delay + +Prevent specific requests from triggering slot delay adjustments +================================================================ + +.. versionadded:: 2.12.0 + +AutoThrottle adjusts the delay of download slots based on the latencies of +responses that belong to that download slot. The only exceptions are non-200 +responses, which are only taken into account to increase that delay, but +ignored if they would decrease that delay. + +You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to +``True`` in any request to prevent its response latency from impacting the +delay of its download slot: + +.. code-block:: python + + from scrapy import Request + + Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) + +Note, however, that AutoThrottle still determines the starting delay of every +download slot. If you want AutoThrottle not to impact a download slot at all, +in addition to setting this meta key in all requests that use that download +slot, you might want to set a custom value for the ``delay`` attribute of that +download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. + Settings ======== @@ -91,7 +122,6 @@ The settings used to control the AutoThrottle extension are: * :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` * :setting:`AUTOTHROTTLE_DEBUG` * :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -* :setting:`CONCURRENT_REQUESTS_PER_IP` * :setting:`DOWNLOAD_DELAY` For more information see :ref:`autothrottle-algorithm`. @@ -131,7 +161,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote -websites. +websites. It must be higher than ``0.0``. By default, AutoThrottle adjusts the delay to send a single concurrent request to each of the remote websites. Set this option to @@ -139,12 +169,10 @@ a higher value (e.g. ``2.0``) to increase the throughput and the load on remote servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value (e.g. ``0.5``) makes the crawler more conservative and polite. -Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected +Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is still respected when AutoThrottle extension is enabled. This means that if ``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, the crawler won't reach this number of concurrent requests. At every given time point Scrapy can be sending more or less concurrent diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 0643df6a6..e8ddec00c 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -24,7 +24,8 @@ You should see an output like this:: 'scrapy.extensions.telnet.TelnetConsole', 'scrapy.extensions.corestats.CoreStats'] 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled downloader middlewares: - ['scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + ['scrapy.downloadermiddlewares.offsite.OffsiteMiddleware', + 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', @@ -37,7 +38,6 @@ You should see an output like this:: 'scrapy.downloadermiddlewares.stats.DownloaderStats'] 2016-12-16 21:18:49 [scrapy.middleware] INFO: Enabled spider middlewares: ['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', 'scrapy.spidermiddlewares.referer.RefererMiddleware', 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', 'scrapy.spidermiddlewares.depth.DepthMiddleware'] @@ -83,4 +83,4 @@ and how well it's written. Use scrapy-bench_ for more complex benchmarking. -.. _scrapy-bench: https://github.com/scrapy/scrapy-bench \ No newline at end of file +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 8be89feb2..cace1f883 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -41,19 +41,6 @@ efficient broad crawl. .. _broad-crawls-scheduler-priority-queue: -Use the right :setting:`SCHEDULER_PRIORITY_QUEUE` -================================================= - -Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. -It works best during single-domain crawl. It does not work well with crawling -many different domains in parallel - -To apply the recommended priority queue use: - -.. code-block:: python - - SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue" - .. _broad-crawls-concurrency: Increase concurrency @@ -61,12 +48,7 @@ Increase concurrency Concurrency is the number of requests that are processed in parallel. There is a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per -IP (:setting:`CONCURRENT_REQUESTS_PER_IP`). - -.. note:: The scheduler priority queue :ref:`recommended for broad crawls - ` does not support - :setting:`CONCURRENT_REQUESTS_PER_IP`. +can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much @@ -116,7 +98,7 @@ Reduce log level When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you -should not use ``DEBUG`` log level when preforming large broad crawls in +should not use ``DEBUG`` log level when performing large broad crawls in production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. @@ -143,7 +125,7 @@ To disable cookies use: Disable retries =============== -Retrying failed HTTP requests can slow down the crawls substantially, specially +Retrying failed HTTP requests can slow down the crawls substantially, especially when sites causes are very slow (or fail) to respond, thus causing a timeout error which gets retried many times, unnecessarily, preventing crawler capacity to be reused for other domains. @@ -182,32 +164,6 @@ To disable redirects use: REDIRECT_ENABLED = False -Enable crawling of "Ajax Crawlable Pages" -========================================= - -Some pages (up to 1%, based on empirical data from year 2013) declare -themselves as `ajax crawlable`_. This means they provide plain HTML -version of content that is usually available only via AJAX. -Pages can indicate it in two ways: - -1) by using ``#!`` in URL - this is the default way; -2) by using a special meta tag - this way is used on - "main", "index" website pages. - -Scrapy handles (1) automatically; to handle (2) enable -:ref:`AjaxCrawlMiddleware `: - -.. code-block:: python - - AJAXCRAWL_ENABLED = True - -When doing broad crawls it's common to crawl a lot of "index" web pages; -AjaxCrawlMiddleware helps to crawl them correctly. -It is turned OFF by default because it has some performance overhead, -and enabling it for focused crawls doesn't make much sense. - -.. _ajax crawlable: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started - .. _broad-crawls-bfo: Crawl in BFO order diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 1d37895c2..50da4593a 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,7 +6,7 @@ Command line tool ================= -Scrapy is controlled through the ``scrapy`` command-line tool, to be referred +Scrapy is controlled through the ``scrapy`` command-line tool, to be referred to here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -114,8 +114,8 @@ some usage help and the available commands:: scrapy [options] [args] Available commands: - crawl Run a spider fetch Fetch a URL using the Scrapy downloader + runspider Run a spider from a Python file, no project required [...] The first line will print the currently active project if you're inside a @@ -163,8 +163,8 @@ information on which commands must be run from inside projects, and which not. Also keep in mind that some commands may have slightly different behaviours when running them from inside projects. For example, the fetch command will use -spider-overridden behaviours (such as the ``user_agent`` attribute to override -the user-agent) if the url being fetched is associated with some specific +spider-overridden behaviours (such as the ``custom_settings`` attribute to +override settings) if the url being fetched is associated with some specific spider. This is intentional, as the ``fetch`` command is meant to be used to check how spiders are downloading pages. @@ -185,8 +185,8 @@ And you can see all available commands with:: There are two kinds of commands, those that only work from inside a Scrapy project (Project-specific commands) and those that also work without an active -Scrapy project (Global commands), though they may behave slightly different -when running from inside a project (as they would use the project overridden +Scrapy project (Global commands), though they may behave slightly differently +when run from inside a project (as they would use the project overridden settings). Global commands: @@ -199,6 +199,7 @@ Global commands: * :command:`fetch` * :command:`view` * :command:`version` +* :command:`bench` Project-only commands: @@ -207,7 +208,6 @@ Project-only commands: * :command:`list` * :command:`edit` * :command:`parse` -* :command:`bench` .. command:: startproject @@ -233,10 +233,7 @@ genspider * Syntax: ``scrapy genspider [-t template] `` * Requires project: *no* -.. versionadded:: 2.6.0 - The ability to pass a URL instead of a domain. - -Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. +Creates a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. Usage example:: @@ -253,7 +250,7 @@ Usage example:: $ scrapy genspider -t crawl scrapyorg scrapy.org Created spider 'scrapyorg' using template 'crawl' -This is just a convenience shortcut command for creating spiders based on +This is just a convenient shortcut command for creating spiders based on pre-defined templates, but certainly not the only way to create spiders. You can just create the spider source code files yourself, instead of using this command. @@ -266,7 +263,9 @@ crawl * Syntax: ``scrapy crawl `` * Requires project: *yes* -Start crawling using a spider. +Start crawling using the spider with the given :attr:`~scrapy.Spider.name`, +which must be one of those that :command:`list` reports. To run a spider from a +file instead, use :command:`runspider`. Supported options: @@ -274,11 +273,9 @@ Supported options: * ``-a NAME=VALUE``: set a spider argument (may be repeated) -* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) +* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout). To define the output format, set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) -* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) - -* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` +* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) Usage examples:: @@ -291,9 +288,6 @@ Usage examples:: $ scrapy crawl -O myfile:json myspider [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] - $ scrapy crawl -o myfile -t csv myspider - [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] - .. command:: check check @@ -317,11 +311,25 @@ Usage examples:: * parse_item $ scrapy check - [FAILED] first_spider:parse_item - >>> 'RetailPricex' field is missing + F.F. + ====================================================================== + FAIL: [first_spider] parse (@returns post-hook) + ---------------------------------------------------------------------- + Traceback (most recent call last): + ... + scrapy.exceptions.ContractFail: Returned 92 requests, expected 0..4 - [FAILED] first_spider:parse - >>> Returned 92 requests, expected 0..4 + ====================================================================== + FAIL: [first_spider] parse_item (@scrapes post-hook) + ---------------------------------------------------------------------- + Traceback (most recent call last): + ... + scrapy.exceptions.ContractFail: Missing fields: RetailPricex + + ---------------------------------------------------------------------- + Ran 4 contracts in 0.174s + + FAILED (failures=2) .. skip: end @@ -353,7 +361,7 @@ edit Edit the given spider using the editor defined in the ``EDITOR`` environment variable or (if unset) the :setting:`EDITOR` setting. -This command is provided only as a convenience shortcut for the most common +This command is provided only as a convenient shortcut for the most common case, the developer is of course free to choose any tool or IDE to write and debug spiders. @@ -372,7 +380,7 @@ fetch Downloads the given URL using the Scrapy downloader and writes the contents to standard output. -The interesting thing about this command is that it fetches the page how the +The interesting thing about this command is that it fetches the page the way the spider would download it. For example, if the spider has a ``USER_AGENT`` attribute which overrides the User Agent, it will use that one. @@ -385,7 +393,7 @@ Supported options: * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider -* ``--headers``: print the response's HTTP headers instead of the response's body +* ``--headers``: print the request's and response's HTTP headers instead of the response's body * ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) @@ -395,15 +403,19 @@ Usage examples:: [ ... html content here ... ] $ scrapy fetch --nolog --headers http://www.example.com/ - {'Accept-Ranges': ['bytes'], - 'Age': ['1263 '], - 'Connection': ['close '], - 'Content-Length': ['596'], - 'Content-Type': ['text/html; charset=UTF-8'], - 'Date': ['Wed, 18 Aug 2010 23:59:46 GMT'], - 'Etag': ['"573c1-254-48c9c87349680"'], - 'Last-Modified': ['Fri, 30 Jul 2010 15:30:18 GMT'], - 'Server': ['Apache/2.2.3 (CentOS)']} + > Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + > Accept-Language: en + > User-Agent: Scrapy/2.16.0 (+https://scrapy.org) + > Accept-Encoding: gzip, deflate, br + > + < Date: Wed, 08 Jul 2026 06:15:01 GMT + < Content-Type: text/html + < Server: cloudflare + < Last-Modified: Wed, 01 Jul 2026 17:50:18 GMT + < Allow: GET, HEAD + < Cf-Cache-Status: HIT + < Age: 8184 + < Cf-Ray: a17cf3b80eddf141-DME .. command:: view @@ -484,7 +496,7 @@ Supported options: * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider -* ``--a NAME=VALUE``: set spider argument (may be repeated) +* ``-a NAME=VALUE``: set spider argument (may be repeated) * ``--callback`` or ``-c``: spider method to use as callback for parsing the response @@ -495,7 +507,7 @@ Supported options: * ``--cbkwargs``: additional keyword arguments that will be passed to the callback. This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}' -* ``--pipelines``: process items through pipelines +* ``--pipelines``: :ref:`process items through pipelines ` * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` rules to discover the callback (i.e. spider method) to use for parsing the @@ -514,8 +526,6 @@ Supported options: * ``--output`` or ``-o``: dump scraped items to a file - .. versionadded:: 2.3 - .. skip: start Usage example:: @@ -563,8 +573,9 @@ runspider * Syntax: ``scrapy runspider `` * Requires project: *no* -Run a spider self-contained in a Python file, without having to create a -project. +Run the spider defined in the given Python file, without requiring a project. + +Supported options: the same as :command:`crawl`. Example usage:: @@ -592,6 +603,47 @@ bench Run a quick benchmark test. :ref:`benchmarking`. +.. _topics-commands-crawlerprocess: + +Commands that run a crawl +========================= + +Many commands need to run a crawl of some kind, running either a user-provided +spider or a special internal one: + +* :command:`bench` +* :command:`check` +* :command:`crawl` +* :command:`fetch` +* :command:`parse` +* :command:`runspider` +* :command:`shell` +* :command:`view` + +They use an internal instance of :class:`scrapy.crawler.AsyncCrawlerProcess` or +:class:`scrapy.crawler.CrawlerProcess` for this. In most cases this detail +shouldn't matter to the user running the command, but when the user :ref:`needs +a non-default Twisted reactor `, it may be important. + +Scrapy decides which of these two classes to use based on the value of the +:setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED` settings. +With :setting:`TWISTED_REACTOR_ENABLED` set to ``False`` it will use +:class:`~scrapy.crawler.AsyncCrawlerProcess`. Otherwise, if the +:setting:`TWISTED_REACTOR` value is the default one +(``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``), +:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise +:class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings +` are not taken into account when doing this, as they are +loaded after this decision is made. This may cause an error if the +project-level setting is set to :ref:`the asyncio reactor ` +(:ref:`explicitly ` or :ref:`by using the Scrapy default +`) and :ref:`the setting of the spider being run +` is set to :ref:`a different one `, because +:class:`~scrapy.crawler.AsyncCrawlerProcess` only supports the asyncio reactor. +In this case you should set the :setting:`FORCE_CRAWLER_PROCESS` setting to +``True`` (at the project level or via the command line) so that Scrapy uses +:class:`~scrapy.crawler.CrawlerProcess` which supports all reactors. + Custom project commands ======================= @@ -616,6 +668,8 @@ Example: COMMANDS_MODULE = "mybot.commands" +.. note:: This is a :ref:`pre-crawler setting `. + .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html Register commands via setup.py entry points diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 478dd9647..354375577 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -4,47 +4,106 @@ Components ========== -A Scrapy component is any class whose objects are created using -:func:`scrapy.utils.misc.create_instance`. +A Scrapy component is any class whose objects are built using +:func:`~scrapy.utils.misc.build_from_crawler`. That includes the classes that you may assign to the following settings: -- :setting:`DNS_RESOLVER` - -- :setting:`DOWNLOAD_HANDLERS` - -- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` - -- :setting:`DOWNLOADER_MIDDLEWARES` - -- :setting:`DUPEFILTER_CLASS` - -- :setting:`EXTENSIONS` - -- :setting:`FEED_EXPORTERS` - -- :setting:`FEED_STORAGES` - -- :setting:`ITEM_PIPELINES` - -- :setting:`SCHEDULER` - -- :setting:`SCHEDULER_DISK_QUEUE` - -- :setting:`SCHEDULER_MEMORY_QUEUE` - -- :setting:`SCHEDULER_PRIORITY_QUEUE` - -- :setting:`SPIDER_MIDDLEWARES` +- :setting:`ADDONS` +- :setting:`DOWNLOAD_HANDLERS` +- :setting:`DOWNLOADER_MIDDLEWARES` +- :setting:`DUPEFILTER_CLASS` +- :setting:`EXTENSIONS` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_STORAGES` +- :setting:`ITEM_PIPELINES` +- :setting:`SCHEDULER` +- :setting:`SCHEDULER_DISK_QUEUE` +- :setting:`SCHEDULER_MEMORY_QUEUE` +- :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` +- :setting:`SCHEDULER_START_MEMORY_QUEUE` +- :setting:`SPIDER_MIDDLEWARES` +- :setting:`TWISTED_DNS_RESOLVER` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to modify their behavior. +.. _from-crawler: + +Initializing from the crawler +============================= + +Any Scrapy component may optionally define the following class method: + +.. classmethod:: from_crawler(cls, crawler: scrapy.crawler.Crawler, *args, **kwargs) + + Return an instance of the component based on *crawler*. + + *args* and *kwargs* are component-specific arguments that some components + receive. However, most components do not get any arguments, and instead + :ref:`use settings `. + + If a component class defines this method, this class method is called to + create any instance of the component. + + The *crawler* object provides access to all Scrapy core components like + :ref:`settings ` and :ref:`signals `, + allowing the component to access them and hook its functionality into + Scrapy. + +.. _component-settings: + +Settings +======== + +Components can be configured through :ref:`settings `. + +Components can read any setting from the +:attr:`~scrapy.crawler.Crawler.settings` attribute of the +:class:`~scrapy.crawler.Crawler` object they can :ref:`get for initialization +`. That includes both built-in and custom settings. + +For example: + +.. code-block:: python + + class MyExtension: + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + return cls(settings.getbool("LOG_ENABLED")) + + def __init__(self, log_is_enabled=False): + if log_is_enabled: + print("log is enabled!") + +Components do not need to declare their custom settings programmatically. +However, they should document them, so that users know they exist and how to +use them. + +It is a good practice to prefix custom settings with the name of the component, +to avoid collisions with custom settings of other existing (or future) +components. For example, an extension called ``WarcCaching`` could prefix its +custom settings with ``WARC_CACHING_``. + +Another good practice, mainly for components meant for :ref:`component priority +dictionaries `, is to provide a boolean setting +called ``_ENABLED`` (e.g. ``WARC_CACHING_ENABLED``) to allow toggling +that component on and off without changing the component priority dictionary +setting. You can usually check the value of such a setting during +initialization, and if ``False``, raise +:exc:`~scrapy.exceptions.NotConfigured`. + +When choosing a name for a custom setting, it is also a good idea to have a +look at the names of :ref:`built-in settings `, to try to +maintain consistency with them. + .. _enforce-component-requirements: -Enforcing component requirements -================================ +Enforcing requirements +====================== Sometimes, your components may only be intended to work under certain conditions. For example, they may require a minimum version of Scrapy to work as @@ -58,8 +117,8 @@ In the case of :ref:`downloader middlewares `, :ref:`extensions `, :ref:`item pipelines `, and :ref:`spider middlewares `, you should raise -:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a -parameter to the exception so that it is printed in the logs, for the user to +:exc:`~scrapy.exceptions.NotConfigured`, passing a description of the issue as +a parameter to the exception so that it is printed in the logs, for the user to see. For other components, feel free to raise whatever other exception feels right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy version mismatch, while :exc:`ValueError` may be better if the issue is the @@ -84,3 +143,15 @@ If your requirement is a minimum Scrapy version, you may use f"method of spider middlewares as an asynchronous " f"generator." ) + +API reference +============= + +The following function can be used to create an instance of a component class: + +.. autofunction:: scrapy.utils.misc.build_from_crawler + +The following function can also be useful when implementing a component, to +report the import path of the component class, e.g. when reporting problems: + +.. autofunction:: scrapy.utils.python.global_object_name diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 2d61026e9..df67bee02 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -20,45 +20,25 @@ following example: This function parses a sample response. Some contracts are mingled with this docstring. - @url http://www.amazon.com/s?field-keywords=selfish+gene + @url http://www.example.com/s?field-keywords=selfish+gene @returns items 1 16 @returns requests 0 0 @scrapes Title Author Year Price """ -This callback is tested using three built-in contracts: +You can use the following contracts: .. module:: scrapy.contracts.default -.. class:: UrlContract +.. autoclass:: UrlContract - This contract (``@url``) sets the sample URL used when checking other - contract conditions for this spider. This contract is mandatory. All - callbacks lacking this contract are ignored when running the checks:: +.. autoclass:: CallbackKeywordArgumentsContract - @url url +.. autoclass:: MetadataContract -.. class:: CallbackKeywordArgumentsContract +.. autoclass:: ReturnsContract - This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` - attribute for the sample request. It must be a valid JSON dictionary. - :: - - @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} - -.. class:: ReturnsContract - - This contract (``@returns``) sets lower and upper bounds for the items and - requests returned by the spider. The upper bound is optional:: - - @returns item(s)|request(s) [min [max]] - -.. class:: ScrapesContract - - This contract (``@scrapes``) checks that all the items returned by the - callback have the specified fields:: - - @scrapes field_1 field_2 ... +.. autoclass:: ScrapesContract Use the :command:`check` command to run the contract checks. @@ -81,30 +61,16 @@ override three methods: .. module:: scrapy.contracts -.. class:: Contract(method, *args) +.. autoclass:: Contract - :param method: callback function to which the contract is associated - :type method: collections.abc.Callable + .. automethod:: adjust_request_args - :param args: list of arguments passed into the docstring (whitespace - separated) - :type args: list - - .. method:: Contract.adjust_request_args(args) - - This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.Request` is used by default, - but this can be changed with the ``request_cls`` attribute. - If multiple contracts in chain have this attribute defined, the last one is used. - - Must return the same or a modified version of it. - - .. method:: Contract.pre_process(response) + .. method:: pre_process(response) This allows hooking in various checks on the response received from the sample request, before it's being passed to the callback. - .. method:: Contract.post_process(output) + .. method:: post_process(output) This allows processing the output of the callback. Iterators are converted to lists before being passed to this hook. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a65bab3ca..b7ddb0a57 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -4,10 +4,9 @@ Coroutines ========== -.. versionadded:: 2.0 +Scrapy :ref:`supports ` the :ref:`coroutine syntax ` +(i.e. ``async def``). -Scrapy has :ref:`partial support ` for the -:ref:`coroutine syntax `. .. _coroutine-support: @@ -17,14 +16,14 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): -- :class:`~scrapy.Request` callbacks. +- The :meth:`~scrapy.Spider.start` spider method, which *must* be + defined as an :term:`asynchronous generator`. - If you are using any custom or third-party :ref:`spider middleware - `, see :ref:`sync-async-spider-middleware`. + .. versionadded:: 2.13 - .. versionchanged:: 2.7 - Output of async callbacks is now processed asynchronously instead of - collecting all of it first. +- :class:`~scrapy.Request` :ref:`callbacks `, which may + also be defined as :term:`asynchronous generators `. - The :meth:`process_item` method of :ref:`item pipelines `. @@ -37,19 +36,102 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): methods of :ref:`downloader middlewares `. -- :ref:`Signal handlers that support deferreds `. - - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` - method of :ref:`spider middlewares `. + method of :ref:`spider middlewares `, which + *must* be defined as an :term:`asynchronous generator` except in + :ref:`universal spider middlewares `. - It must be defined as an :term:`asynchronous generator`. The input - ``result`` parameter is an :term:`asynchronous iterable`. +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method + of :ref:`spider middlewares `, which *must* be + defined as an :term:`asynchronous generator`. - See also :ref:`sync-async-spider-middleware` and - :ref:`universal-spider-middleware`. + .. versionadded:: 2.13 + +- :ref:`Signal handlers that support deferreds `. + +- Methods of :ref:`download handlers `. + + .. versionadded:: 2.14 + + +.. _coroutine-deferred-apis: + +Using Deferred-based APIs +========================= + +In addition to native coroutine APIs Scrapy has some APIs that return a +:class:`~twisted.internet.defer.Deferred` object or take a user-supplied +function that returns a :class:`~twisted.internet.defer.Deferred` object. These +APIs are also asynchronous but don't yet support native ``async def`` syntax. +In the future we plan to add support for the ``async def`` syntax to these APIs +or replace them with other APIs where changing the existing ones isn't +possible. + +These APIs have a coroutine-based implementation and a Deferred-based one: + +- :class:`scrapy.crawler.Crawler`: + + - :meth:`~scrapy.crawler.Crawler.crawl_async` (coroutine-based) and + :meth:`~scrapy.crawler.Crawler.crawl` (Deferred-based): the former + may be inconvenient to use in Deferred-based code so both are available, + this may change in a future Scrapy version. + +- :class:`scrapy.crawler.AsyncCrawlerRunner` and its subclass + :class:`scrapy.crawler.AsyncCrawlerProcess` (coroutine-based) and + :class:`scrapy.crawler.CrawlerRunner` and its subclass + :class:`scrapy.crawler.CrawlerProcess` (Deferred-based): the former + doesn't support non-default reactors and so the latter should be used + with those. + +The following user-supplied methods can return +:class:`~twisted.internet.defer.Deferred` objects (the methods that can also +return coroutines are listed in :ref:`coroutine-support`): + +- Custom downloader implementations (see :setting:`DOWNLOADER`): + + - ``fetch()`` + +- Custom scheduler implementations (see :setting:`SCHEDULER`): + + - :meth:`~scrapy.core.scheduler.BaseScheduler.open` + + - :meth:`~scrapy.core.scheduler.BaseScheduler.close` + +- Custom dupefilters (see :setting:`DUPEFILTER_CLASS`): + + - ``open()`` + + - ``close()`` + +- Custom feed storages (see :setting:`FEED_STORAGES`): + + - ``store()`` + +- Subclasses of :class:`scrapy.pipelines.media.MediaPipeline`: + + - ``media_to_download()`` + + - ``item_completed()`` + +- Custom storages used by subclasses of + :class:`scrapy.pipelines.files.FilesPipeline`: + + - ``persist_file()`` + + - ``stat_file()`` + +In most cases you can use these APIs in code that otherwise uses coroutines, by +wrapping a :class:`~twisted.internet.defer.Deferred` object into a +:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for +more information about this. + +For example: a custom scheduler needs to define an ``open()`` method that can +return a :class:`~twisted.internet.defer.Deferred` object. You can write a +method that works with Deferreds and returns one directly, or you can write a +coroutine and convert it into a function that returns a Deferred with +:func:`~scrapy.utils.defer.deferred_f_from_coro_f`. - .. versionadded:: 2.7 General usage ============= @@ -71,7 +153,7 @@ shorter and cleaner: adapter["field"] = data return item - def process_item(self, item, spider): + def process_item(self, item): adapter = ItemAdapter(item) dfd = db.get_some_data(adapter["id"]) dfd.addCallback(self._update_item, item) @@ -85,7 +167,7 @@ becomes: class DbPipeline: - async def process_item(self, item, spider): + async def process_item(self, item): adapter = ItemAdapter(item) adapter["field"] = await db.get_some_data(adapter["id"]) return item @@ -123,13 +205,16 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: -* requesting data from websites, databases and other services (in callbacks, - pipelines and middlewares); +* requesting data from websites, databases and other services (in + :meth:`~scrapy.Spider.start`, callbacks, pipelines and + middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); -* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download` - (see :ref:`the screenshot pipeline example`). +* calling asynchronous Scrapy methods like + :meth:`ExecutionEngine.download_async() + ` (see :ref:`the + screenshot pipeline example `). .. _aio-libs: https://github.com/aio-libs @@ -145,7 +230,6 @@ within a spider callback: .. code-block:: python from scrapy import Spider, Request - from scrapy.utils.defer import maybe_deferred_to_future class SingleRequestSpider(Spider): @@ -154,8 +238,9 @@ within a spider callback: async def parse(self, response, **kwargs): additional_request = Request("https://example.org/price") - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) + additional_response = await self.crawler.engine.download_async( + additional_request + ) yield { "h1": response.css("h1").get(), "price": additional_response.css("#price").get(), @@ -165,9 +250,9 @@ You can also send multiple requests in parallel: .. code-block:: python + import asyncio + from scrapy import Spider, Request - from scrapy.utils.defer import maybe_deferred_to_future - from twisted.internet.defer import DeferredList class MultipleRequestsSpider(Spider): @@ -179,108 +264,13 @@ You can also send multiple requests in parallel: Request("https://example.com/price"), Request("https://example.com/color"), ] - deferreds = [] + tasks = [] for r in additional_requests: - deferred = self.crawler.engine.download(r) - deferreds.append(deferred) - responses = await maybe_deferred_to_future(DeferredList(deferreds)) + task = self.crawler.engine.download_async(r) + tasks.append(task) + responses = await asyncio.gather(*tasks) yield { "h1": response.css("h1::text").get(), - "price": responses[0][1].css(".price::text").get(), - "price2": responses[1][1].css(".color::text").get(), + "price": responses[0].css(".price::text").get(), + "color": responses[1].css(".color::text").get(), } - - -.. _sync-async-spider-middleware: - -Mixing synchronous and asynchronous spider middlewares -====================================================== - -.. versionadded:: 2.7 - -The output of a :class:`~scrapy.Request` callback is passed as the ``result`` -parameter to the -:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method -of the first :ref:`spider middleware ` from the -:ref:`list of active spider middlewares `. -Then the output of that ``process_spider_output`` method is passed to the -``process_spider_output`` method of the next spider middleware, and so on for -every active spider middleware. - -Scrapy supports mixing :ref:`coroutine methods ` and synchronous methods -in this chain of calls. - -However, if any of the ``process_spider_output`` methods is defined as a -synchronous method, and the previous ``Request`` callback or -``process_spider_output`` method is a coroutine, there are some drawbacks to -the asynchronous-to-synchronous conversion that Scrapy does so that the -synchronous ``process_spider_output`` method gets a synchronous iterable as its -``result`` parameter: - -- The whole output of the previous ``Request`` callback or - ``process_spider_output`` method is awaited at this point. - -- If an exception raises while awaiting the output of the previous - ``Request`` callback or ``process_spider_output`` method, none of that - output will be processed. - - This contrasts with the regular behavior, where all items yielded before - an exception raises are processed. - -Asynchronous-to-synchronous conversions are supported for backward -compatibility, but they are deprecated and will stop working in a future -version of Scrapy. - -To avoid asynchronous-to-synchronous conversions, when defining ``Request`` -callbacks as coroutine methods or when using spider middlewares whose -``process_spider_output`` method is an :term:`asynchronous generator`, all -active spider middlewares must either have their ``process_spider_output`` -method defined as an asynchronous generator or :ref:`define a -process_spider_output_async method `. - -.. note:: When using third-party spider middlewares that only define a - synchronous ``process_spider_output`` method, consider - :ref:`making them universal ` through - :ref:`subclassing `. - - -.. _universal-spider-middleware: - -Universal spider middlewares -============================ - -.. versionadded:: 2.7 - -To allow writing a spider middleware that supports asynchronous execution of -its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding -:ref:`asynchronous-to-synchronous conversions `) -while maintaining support for older Scrapy versions, you may define -``process_spider_output`` as a synchronous method and define an -:term:`asynchronous generator` version of that method with an alternative name: -``process_spider_output_async``. - -For example: - -.. code-block:: python - - class UniversalSpiderMiddleware: - def process_spider_output(self, response, result, spider): - for r in result: - # ... do something with r - yield r - - async def process_spider_output_async(self, response, result, spider): - async for r in result: - # ... do something with r - yield r - -.. note:: This is an interim measure to allow, for a time, to write code that - works in Scrapy 2.7 and later without requiring - asynchronous-to-synchronous conversions, and works in earlier Scrapy - versions as well. - - In some future version of Scrapy, however, this feature will be - deprecated and, eventually, in a later version of Scrapy, this - feature will be removed, and all spider middlewares will be expected - to define their ``process_spider_output`` method as an asynchronous - generator. diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 49c5b0410..988e37bbd 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -125,25 +125,15 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see See also: :ref:`topics-shell-inspect-response`. + Open in browser =============== Sometimes you just want to see how a certain response looks in a browser, you -can use the ``open_in_browser`` function for that. Here is an example of how -you would use it: +can use the :func:`~scrapy.utils.response.open_in_browser` function for that: -.. code-block:: python +.. autofunction:: scrapy.utils.response.open_in_browser - from scrapy.utils.response import open_in_browser - - - def parse_details(self, response): - if "item name" not in response.body: - open_in_browser(response) - -``open_in_browser`` will open a browser with the response received by Scrapy at -that point, adjusting the `base tag`_ so that images and styles are displayed -properly. Logging ======= @@ -163,8 +153,6 @@ available in all future runs should they be necessary again: For more information, check the :ref:`topics-logging` section. -.. _base tag: https://www.w3schools.com/tags/tag_base.asp - .. _debug-vscode: Visual Studio Code diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index 961d6dc01..f3515b4be 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -54,6 +54,6 @@ just like ``scrapyd-deploy``. .. _scrapyd-client: https://github.com/scrapy/scrapyd-client .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html .. _shub: https://shub.readthedocs.io/en/latest/ -.. _Zyte: https://zyte.com/ +.. _Zyte: https://www.zyte.com/ .. _Zyte Scrapy Cloud: https://www.zyte.com/scrapy-cloud/ .. _Zyte Scrapy Cloud documentation: https://docs.zyte.com/scrapy-cloud.html diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index a15ee1059..a5ffe00f1 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -246,7 +246,6 @@ also request each page to get every quote on the site: .. code-block:: python import scrapy - import json class QuoteSpider(scrapy.Spider): @@ -256,7 +255,7 @@ also request each page to get every quote on the site: start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"] def parse(self, response): - data = json.loads(response.text) + data = response.json() for quote in data["quotes"]: yield {"quote": quote["text"]} if data["has_next"]: @@ -278,9 +277,9 @@ into our ``url``. In more complex websites, it could be difficult to easily reproduce the requests, as we could need to add ``headers`` or ``cookies`` to make it work. -In those cases you can export the requests in `cURL `_ +In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the -:meth:`~scrapy.Request.from_curl()` method to generate an equivalent +:meth:`~scrapy.Request.from_curl` method to generate an equivalent request: .. code-block:: python @@ -317,4 +316,3 @@ to identifying the correct request and replicating it in your spider. .. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll .. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions - diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst new file mode 100644 index 000000000..e0501c169 --- /dev/null +++ b/docs/topics/download-handlers.rst @@ -0,0 +1,385 @@ +.. _topics-download-handlers: + +================= +Download handlers +================= + +Download handlers are Scrapy :ref:`components ` used to +download :ref:`requests ` and produce responses from +them. + +Using download handlers +======================= + +The :setting:`DOWNLOAD_HANDLERS_BASE` and :setting:`DOWNLOAD_HANDLERS` settings +tell Scrapy which handler is responsible for a given URL scheme. Their values +are merged into a mapping from scheme names to handler classes. When Scrapy +initializes it creates instances of all configured download handlers (except +for :ref:`lazy ones `) and stores them in a similar +mapping. When Scrapy needs to download a request it extracts the scheme from +its URL, finds the handler for this scheme, passes the request to it and gets a +response from it. If there is no handler for the scheme, the request is not +downloaded and a :exc:`~scrapy.exceptions.NotSupported` exception is raised. + +The :setting:`DOWNLOAD_HANDLERS_BASE` setting contains the default mapping of +handlers. You can use the :setting:`DOWNLOAD_HANDLERS` setting to add handlers +for additional schemes and to replace or disable default ones: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + # disable support for ftp:// requests + "ftp": None, + # replace the default one for http:// + "http": "my.download_handlers.HttpHandler", + # http:// and https:// are different schemes, + # even though they may use the same handler + "https": "my.download_handlers.HttpHandler", + # support for any custom scheme can be added + "sftp": "my.download_handlers.SftpHandler", + } + +.. seealso:: :ref:`security-unencrypted-protocols` and + :ref:`security-local-resources`, for the security implications of the + default ``http``, ``ftp``, ``file`` and ``data`` handlers. + +Replacing HTTP(S) download handlers +----------------------------------- + +While Scrapy provides a default handler for ``http`` and ``https`` schemes, +users may want to use a different handler, provided by Scrapy or by some +3rd-party package. There are several considerations to keep in mind related to +this. + +First of all, as ``http`` and ``https`` are separate schemes, they need +separate entries in the :setting:`DOWNLOAD_HANDLERS` setting, even though it's +likely that the same handler class will be used for both schemes. + +Additionally, some of the Scrapy settings, like :setting:`DOWNLOAD_MAXSIZE`, +are honored by the default HTTP(S) handler but not necessarily by alternative +ones. The same may apply to other Scrapy features, e.g. the +:signal:`bytes_received` and :signal:`headers_received` signals. + +.. _lazy-download-handlers: + +Lazy instantiation of download handlers +--------------------------------------- + +A download handler can be marked as "lazy" by setting its ``lazy`` class +attribute to ``True``. Such handlers are only instantiated when they need to +download their first request. This may be useful when the instantiation is slow +or requires dependencies that are not always available, and the handler is not +needed on every spider run. For example, :class:`the built-in S3 handler +<.S3DownloadHandler>` is lazy. + +Writing your own download handler +================================= + +A download handler is a :ref:`component ` that defines +the following API: + +.. autoclass:: scrapy.core.downloader.handlers.DownloadHandlerProtocol + :members: + +An optional base class for custom handlers is provided: + +.. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler + :members: + :undoc-members: + :exclude-members: close, download_request, lazy + +.. _download-handlers-exceptions: + +Exceptions raised by download handlers +====================================== + +.. versionadded:: 2.15.0 + +The built-in download handlers raise Scrapy-specific exceptions instead of +implementation-specific ones, so that code that handles these exceptions can be +written in a generic way. We recommend custom download handlers to also use +these exceptions. + +.. autoexception:: scrapy.exceptions.CannotResolveHostError + +.. autoexception:: scrapy.exceptions.DownloadCancelledError + +.. autoexception:: scrapy.exceptions.DownloadConnectionRefusedError + +.. autoexception:: scrapy.exceptions.DownloadFailedError + +.. autoexception:: scrapy.exceptions.DownloadTimeoutError + +.. autoexception:: scrapy.exceptions.ResponseDataLossError + +.. autoexception:: scrapy.exceptions.UnsupportedURLSchemeError + +.. _download-handlers-ref: + +Built-in HTTP download handlers reference +========================================= + +Scrapy ships several handlers for HTTP and HTTPS requests. While all of them +support basic features, they may differ in support of specific Scrapy features +and settings and HTTP protocol features. See the documentation of specific +handlers and specific settings for more information. Additionally, as the +underlying HTTP client implementations differ between handlers, the behavior of +specific websites may be different when doing the same Scrapy requests but +using different handlers. + +Here is a comparison of some features of the built-in HTTP handlers, see the +individual handler docs for more differences: + +================== ================= ===================== ==================== +Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler +================== ================= ===================== ==================== +Requires asyncio No No Yes +Requires a reactor Yes Yes No +HTTP/1.1 No Yes Yes +HTTP/2 Yes No Yes +TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` +HTTP proxies No Yes Yes +SOCKS proxies No No Yes +================== ================= ===================== ==================== + +You can find additional HTTP download handlers in the +scrapy-download-handlers-incubator_ package. This package is made by the Scrapy +developers and contains experimental handlers that may be included in some +later Scrapy version but can already be used. Please refer to the documentation +of this package for more information. + +.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator + +.. _twisted-http2-handler: + +H2DownloadHandler +----------------- + +.. note:: Requires the :ref:`twisted-http2 ` extra. + +.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler + +| Supported scheme: ``https``. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol +for them. + +It's implemented using :mod:`twisted.web.client` and the ``h2`` library. + +If you want to use this handler you need to replace the default one for the +``https`` scheme: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", + } + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: + + This handler is experimental, and not yet recommended for production + environments. Future Scrapy versions may introduce related changes without + a deprecation period or warning. + +=========================== ================================================ +HTTP proxies No (not implemented) +SOCKS proxies No (not supported by the library) +HTTP/2 Yes +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- No support for HTTP/1.1. + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- No support for the :signal:`bytes_received` and :signal:`headers_received` + signals. + +Known limitations of the HTTP/2 support: + +- No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). + +- No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will fail. + +- No support for `server pushes`_, which are ignored. + +.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 +.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption +.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 + +HTTP11DownloadHandler +--------------------- + +.. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler + +| Supported schemes: ``http``, ``https``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``http://host/path`` and ``https://host/path`` URLs and +uses the HTTP/1.1 protocol for them. + +It's implemented using :mod:`twisted.web.client`. + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +=========================== ================================================ +HTTP proxies Yes +SOCKS proxies No (not supported by the library) +HTTP/2 No (implemented as a separate handler) +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- HTTPS proxies to HTTPS destinations are not supported. + +.. _httpx-handler: + +HttpxDownloadHandler +-------------------- + +.. note:: Requires the :ref:`httpx ` extra. + +.. versionadded:: 2.15.0 + +.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler + +| Supported schemes: ``http``, ``https``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: yes. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``http://host/path`` and ``https://host/path`` URLs and +uses the HTTP/1.1 or HTTP/2 protocol for them. + +It's implemented using the httpx2_ library. + +.. _httpx2: https://httpx2.pydantic.dev/ + +If you want to use this handler you need to replace the default ones for the +``http`` and ``https`` schemes: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + } + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: + + This handler is experimental, and not yet recommended for production + environments. Future Scrapy versions may introduce related changes without + a deprecation period or warning or even remove it altogether. + +=========================== ======================================= +HTTP proxies Yes +SOCKS proxies Yes (SOCKS5) +HTTP/2 Yes +``response.certificate`` DER bytes +Per-request ``bindaddress`` No (not supported by the library) +TLS implementation Standard library ``ssl`` +=========================== ======================================= + +Other limitations: + +- The handler creates a separate connection pool for each proxy URL (due to + limitations of ``httpx``) which may lead to higher resource usage when + using proxy rotation. + +.. setting:: HTTPX_HTTP2_ENABLED + +HTTPX_HTTP2_ENABLED +^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.17.0 + +Default: ``False`` + +Whether to enable HTTP/2 support in this handler. + +Built-in non-HTTP download handlers reference +============================================= + +DataURIDownloadHandler +---------------------- + +.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler + +| Supported scheme: ``data``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports RFC 2397 ``data:content/type;base64,`` data URIs. + +FileDownloadHandler +------------------- + +.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler + +| Supported scheme: ``file``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``file:///path`` local file URIs. It doesn't +support remote files. + +FTPDownloadHandler +------------------ + +.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler + +| Supported scheme: ``ftp``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``ftp://host/path`` FTP URIs. + +It's implemented using :mod:`twisted.protocols.ftp`. + +.. _s3-handler: + +S3DownloadHandler +----------------- + +.. note:: Requires the :ref:`s3 ` extra. + +.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler + +| Supported scheme: ``s3``. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``s3://bucket/path`` S3 URIs. + +It's implemented using the botocore_ library. + +.. _botocore: https://github.com/boto/botocore diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index a8e5b23bf..fcfe7fd29 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -61,26 +61,23 @@ particular setting. See each middleware documentation for more info. Writing your own downloader middleware ====================================== -Each downloader middleware is a Python class that defines one or more of the -methods defined below. - -The main entry point is the ``from_crawler`` class method, which receives a -:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` -object gives you access, for example, to the :ref:`settings `. +Each downloader middleware is a :ref:`component ` that +defines one or more of these methods: .. module:: scrapy.downloadermiddlewares .. class:: DownloaderMiddleware - .. note:: Any of the downloader middleware methods may also return a deferred. + .. note:: Any of the downloader middleware methods may be defined as a + coroutine function (``async def``). - .. method:: process_request(request, spider) + .. method:: process_request(request) This method is called for each request that goes through the download middleware. :meth:`process_request` should either: return ``None``, return a - :class:`~scrapy.Response` object, return a :class:`~scrapy.http.Request` + :class:`~scrapy.http.Response` object, return a :class:`~scrapy.Request` object, or raise :exc:`~scrapy.exceptions.IgnoreRequest`. If it returns ``None``, Scrapy will continue processing this request, executing all @@ -106,10 +103,7 @@ object gives you access, for example, to the :ref:`settings `. :param request: the request being processed :type request: :class:`~scrapy.Request` object - :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.Spider` object - - .. method:: process_response(request, response, spider) + .. method:: process_response(request, response) :meth:`process_response` should either: return a :class:`~scrapy.http.Response` object, return a :class:`~scrapy.Request` object or @@ -133,14 +127,12 @@ object gives you access, for example, to the :ref:`settings `. :param response: the response being processed :type response: :class:`~scrapy.http.Response` object - :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.Spider` object + .. method:: process_exception(request, exception) - .. method:: process_exception(request, exception, spider) - - Scrapy calls :meth:`process_exception` when a download handler - or a :meth:`process_request` (from a downloader middleware) raises an - exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception) + Scrapy calls :meth:`process_exception` when a :ref:`download handler + ` or a :meth:`process_request` (from a + downloader middleware) raises an exception (including an + :exc:`~scrapy.exceptions.IgnoreRequest` exception). :meth:`process_exception` should return: either ``None``, a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object. @@ -164,20 +156,6 @@ object gives you access, for example, to the :ref:`settings `. :param exception: the raised exception :type exception: an ``Exception`` object - :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.Spider` object - - .. method:: from_crawler(cls, crawler) - - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. _topics-downloader-middleware-ref: Built-in downloader middleware reference @@ -313,13 +291,12 @@ DownloadTimeoutMiddleware .. class:: DownloadTimeoutMiddleware This middleware sets the download timeout for requests specified in the - :setting:`DOWNLOAD_TIMEOUT` setting or :attr:`download_timeout` - spider attribute. + :setting:`DOWNLOAD_TIMEOUT` setting. .. note:: - You can also set download timeout per-request using - :reqmeta:`download_timeout` Request.meta key; this is supported + You can also set download timeout per-request using the + :reqmeta:`download_timeout` :attr:`.Request.meta` key; this is supported even when DownloadTimeoutMiddleware is disabled. HttpAuthMiddleware @@ -330,26 +307,15 @@ HttpAuthMiddleware .. class:: HttpAuthMiddleware - This middleware authenticates all requests generated from certain spiders - using `Basic access authentication`_ (aka. HTTP auth). + This middleware authenticates requests using `Basic access authentication`_ + (aka. HTTP auth). - To enable HTTP authentication for a spider, set the ``http_user`` and - ``http_pass`` spider attributes to the authentication data and the - ``http_auth_domain`` spider attribute to the domain which requires this - authentication (its subdomains will be also handled in the same way). - You can set ``http_auth_domain`` to ``None`` to enable the - authentication for all requests but you risk leaking your authentication - credentials to unrelated domains. + Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and + :setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override + the credentials per request via :attr:`~scrapy.Request.meta` keys + :reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`. - .. warning:: - In previous Scrapy versions HttpAuthMiddleware sent the authentication - data with all requests, which is a security problem if the spider - makes requests to several different domains. Currently if the - ``http_auth_domain`` attribute is not set, the middleware will use the - domain of the first request, which will work for some spiders but not - for others. In the future the middleware will produce an error instead. - - Example: + Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`): .. code-block:: python @@ -357,13 +323,70 @@ HttpAuthMiddleware class SomeIntranetSiteSpider(CrawlSpider): - http_user = "someuser" - http_pass = "somepass" - http_auth_domain = "intranet.example.com" name = "intranet.example.com" + custom_settings = { + "HTTPAUTH_USER": "someuser", + "HTTPAUTH_PASS": "somepass", + "HTTPAUTH_DOMAIN": "intranet.example.com", + } # .. rest of the spider code omitted ... + Example using per-request meta: + + .. code-block:: python + + async def start(self): + yield Request( + "https://intranet.example.com/protected/", + meta={ + "http_user": "someuser", + "http_pass": "somepass", + "http_auth_domain": "intranet.example.com", + }, + ) + +.. setting:: HTTPAUTH_USER + +HTTPAUTH_USER +~~~~~~~~~~~~~ + +.. versionadded:: 2.17.0 + +Default: ``""`` + +The username to use for HTTP basic authentication, applied to all requests +whose URL matches :setting:`HTTPAUTH_DOMAIN`. + +.. setting:: HTTPAUTH_PASS + +HTTPAUTH_PASS +~~~~~~~~~~~~~ + +.. versionadded:: 2.17.0 + +Default: ``""`` + +The password to use for HTTP basic authentication. + +.. setting:: HTTPAUTH_DOMAIN + +HTTPAUTH_DOMAIN +~~~~~~~~~~~~~~~ + +.. versionadded:: 2.17.0 + +Default: ``None`` + +The domain (and its subdomains) to which HTTP basic authentication credentials +are sent. Set to ``None`` to send credentials with all requests, but be aware +that this risks leaking credentials to unrelated domains. + +This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER` +or :setting:`HTTPAUTH_PASS` is set. + +.. seealso:: :ref:`security-credential-leakage` + .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication @@ -482,7 +505,7 @@ Filesystem storage backend (default) * ``response_body`` - the plain response body - * ``response_headers`` - the request headers (in raw HTTP format) + * ``response_headers`` - the response headers (in raw HTTP format) * ``meta`` - some metadata of this cache resource in Python ``repr()`` format (grep-friendly format) @@ -524,7 +547,7 @@ defines the methods described below. .. method:: open_spider(spider) This method gets called after a spider has been opened for crawling. It handles - the :signal:`open_spider ` signal. + the :signal:`spider_opened` signal. :param spider: the spider which has been opened :type spider: :class:`~scrapy.Spider` object @@ -532,7 +555,7 @@ defines the methods described below. .. method:: close_spider(spider) This method gets called after a spider has been closed. It handles - the :signal:`close_spider ` signal. + the :signal:`spider_closed` signal. :param spider: the spider which has been closed :type spider: :class:`~scrapy.Spider` object @@ -541,6 +564,10 @@ defines the methods described below. Return response if present in cache, or ``None`` otherwise. + If this method raises an exception, e.g. because the cache entry is + corrupted, the middleware logs a warning and handles the request as a + cache miss. + :param spider: the spider which generated the request :type spider: :class:`~scrapy.Spider` object @@ -568,8 +595,8 @@ In order to use your storage backend, set: HTTPCache middleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :class:`HttpCacheMiddleware` can be configured through the following -settings: +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` can be +configured through the following settings: .. setting:: HTTPCACHE_ENABLED @@ -704,6 +731,8 @@ We assume that the spider will not issue Cache-Control directives in requests unless it actually needs them, so directives in requests are not filtered. +.. _http-compression: + HttpCompressionMiddleware ------------------------- @@ -715,14 +744,12 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is - installed, respectively. + This middleware also supports decoding `brotli-compressed`_ responses with + the :ref:`brotli ` extra, and `zstd-compressed`_ + responses with the :ref:`zstd ` extra. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt -.. _brotli: https://pypi.org/project/Brotli/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt -.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings @@ -749,7 +776,7 @@ HttpProxyMiddleware .. class:: HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the - ``proxy`` meta value for :class:`~scrapy.Request` objects. + :reqmeta:`proxy` meta value for :class:`~scrapy.Request` objects. Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: @@ -758,11 +785,97 @@ HttpProxyMiddleware * ``https_proxy`` * ``no_proxy`` - You can also set the meta key ``proxy`` per-request, to a value like + You can also set the meta key :reqmeta:`proxy` per-request, to a value like ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. +.. note:: + + Handling of this meta key needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. It's currently unsupported by + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`. + +.. note:: + + Usually a proxy URL uses the ``http://`` scheme. More rarely, it uses the + ``https://`` one. While both kinds of proxy URLs can be used with both HTTP + and HTTPS destination URLs, the specifics of the network exchange are + different for all 4 cases and it's possible that HTTPS proxies are fully or + partially unsupported by a given download handler. Currently, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + supports HTTPS proxies only for HTTP destinations. + +.. note:: + + If the download handler supports it, you can use a SOCKS proxy URL (e.g. + ``socks5://username:password@some_proxy_server:port``). + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` + supports SOCKS proxies while other built-in handlers don't. + +HttpProxyMiddleware settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: HTTPPROXY_ENABLED + +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + +.. setting:: HTTPPROXY_AUTH_ENCODING + +HTTPPROXY_AUTH_ENCODING +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"latin-1"`` + +The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + +OffsiteMiddleware +----------------- + +.. module:: scrapy.downloadermiddlewares.offsite + :synopsis: Offsite Middleware + +.. class:: OffsiteMiddleware + + .. versionadded:: 2.11.2 + + Filters out Requests for URLs outside the domains covered by the spider. + + This middleware filters out every request whose host names aren't in the + spider's :attr:`~scrapy.Spider.allowed_domains` attribute. + All subdomains of any domain in the list are also allowed. + E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` + but not ``www2.example.com`` nor ``example.com``. + + When your spider returns a request for a domain not belonging to those + covered by the spider, this middleware will log a debug message similar to + this one:: + + DEBUG: Filtered offsite request to 'offsite.example': + + To avoid filling the log with too much noise, it will only print one of + these messages for each new domain filtered. So, for example, if another + request for ``offsite.example`` is filtered, no log message will be + printed. But if a request for ``other.example`` is filtered, a message + will be printed (but only for the first request filtered). + + If the spider doesn't define an + :attr:`~scrapy.Spider.allowed_domains` attribute, or the + attribute is empty, the offsite middleware will allow all requests. + + .. reqmeta:: allow_offsite + + If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to + ``True`` or :attr:`Request.meta ` has ``allow_offsite`` + set to ``True``, then the OffsiteMiddleware will allow the request even if + its domain is not listed in allowed domains. + RedirectMiddleware ------------------ @@ -838,7 +951,7 @@ REDIRECT_MAX_TIMES Default: ``20`` The maximum number of redirections that will be followed for a single request. -After this maximum, the request's response is returned as is. +If maximum redirections are exceeded, the request is aborted and ignored. MetaRefreshMiddleware --------------------- @@ -876,13 +989,13 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``[]`` +Default: ``["noscript"]`` Meta tags within these tags are ignored. -.. versionchanged:: 2.0 +.. versionchanged:: 2.11.2 The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``['script', 'noscript']`` to ``[]``. + ``[]`` to ``["noscript"]``. .. setting:: METAREFRESH_MAXDELAY @@ -906,17 +1019,6 @@ RetryMiddleware A middleware to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. -Failed pages are collected on the scraping process and rescheduled at the -end, once the spider has finished crawling all regular (non failed) pages. - -The :class:`RetryMiddleware` can be configured through the following -settings (see the settings documentation for more info): - -* :setting:`RETRY_ENABLED` -* :setting:`RETRY_TIMES` -* :setting:`RETRY_HTTP_CODES` -* :setting:`RETRY_EXCEPTIONS` - .. reqmeta:: dont_retry If :attr:`Request.meta ` has ``dont_retry`` key @@ -975,16 +1077,15 @@ RETRY_EXCEPTIONS Default:: [ - 'twisted.internet.defer.TimeoutError', - 'twisted.internet.error.TimeoutError', - 'twisted.internet.error.DNSLookupError', - 'twisted.internet.error.ConnectionRefusedError', + 'scrapy.exceptions.CannotResolveHostError', + 'scrapy.exceptions.DownloadConnectionRefusedError', + 'scrapy.exceptions.DownloadFailedError', + 'scrapy.exceptions.DownloadTimeoutError', + 'scrapy.exceptions.ResponseDataLossError', 'twisted.internet.error.ConnectionDone', 'twisted.internet.error.ConnectError', 'twisted.internet.error.ConnectionLost', - 'twisted.internet.error.TCPTimedOutError', - 'twisted.web.client.ResponseFailed', - IOError, + OSError, 'scrapy.core.downloader.handlers.http11.TunnelError', ] @@ -998,6 +1099,23 @@ has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught exception propagation, see :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. +.. setting:: RETRY_GIVE_UP_LOG_LEVEL + +RETRY_GIVE_UP_LOG_LEVEL +^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.17.0 + +Default: ``"ERROR"`` + +:ref:`Logging level ` used for the message logged when a request +exceeds its retries. + +Can be a level name (e.g. ``"WARNING"``) or a number (e.g. ``logging.WARNING`` +or ``30``). + +See also: :reqmeta:`give_up_log_level`, :func:`get_retry_request`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST @@ -1039,7 +1157,6 @@ RobotsTxtMiddleware * :ref:`Protego ` (default) * :ref:`RobotFileParser ` - * :ref:`Reppy ` * :ref:`Robotexclusionrulesparser ` You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` @@ -1060,7 +1177,7 @@ Parsers vary in several aspects: * Support for wildcard matching -* Usage of `length based rule `_: +* Usage of `length based rule `_: in particular for ``Allow`` and ``Disallow`` directives, where the most specific rule based on the length of the path trumps the less specific (shorter) rule @@ -1078,7 +1195,7 @@ Based on `Protego `_: * implemented in Python * is compliant with `Google's Robots.txt Specification - `_ + `_ * supports wildcard matching @@ -1098,9 +1215,9 @@ Based on :class:`~urllib.robotparser.RobotFileParser`: * is compliant with `Martijn Koster's 1996 draft specification `_ -* lacks support for wildcard matching +* lacks support for wildcard matching (before Python 3.14.5) -* doesn't use the length based rule +* doesn't use the length based rule (before Python 3.14.5) It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0. @@ -1108,42 +1225,12 @@ In order to use this parser, set: * :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser`` -.. _reppy-parser: - -Reppy parser -~~~~~~~~~~~~ - -Based on `Reppy `_: - -* is a Python wrapper around `Robots Exclusion Protocol Parser for C++ - `_ - -* is compliant with `Martijn Koster's 1996 draft specification - `_ - -* supports wildcard matching - -* uses the length based rule - -Native implementation, provides better speed than Protego. - -In order to use this parser: - -* Install `Reppy `_ by running ``pip install reppy`` - - .. warning:: `Upstream issue #122 - `_ prevents reppy usage in Python 3.9+. - -* Set :setting:`ROBOTSTXT_PARSER` setting to - ``scrapy.robotstxt.ReppyRobotParser`` - - .. _rerp-parser: Robotexclusionrulesparser ~~~~~~~~~~~~~~~~~~~~~~~~~ -Based on `Robotexclusionrulesparser `_: +Based on `Robotexclusionrulesparser `_: * implemented in Python @@ -1156,8 +1243,7 @@ Based on `Robotexclusionrulesparser `_: In order to use this parser: -* Install `Robotexclusionrulesparser `_ by running - ``pip install robotexclusionrulesparser`` +* Install the :ref:`robotparser ` extra. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.RerpRobotParser`` @@ -1201,64 +1287,8 @@ UserAgentMiddleware .. class:: UserAgentMiddleware - Middleware that allows spiders to override the default user agent. - - In order for a spider to override the default user agent, its ``user_agent`` - attribute must be set. - -.. _ajaxcrawl-middleware: - -AjaxCrawlMiddleware -------------------- - -.. module:: scrapy.downloadermiddlewares.ajaxcrawl - -.. class:: AjaxCrawlMiddleware - - Middleware that finds 'AJAX crawlable' page variants based - on meta-fragment html tag. See - https://developers.google.com/search/docs/ajax-crawling/docs/getting-started - for more info. - - .. note:: - - Scrapy finds 'AJAX crawlable' pages for URLs like - ``'http://example.com/!#foo=bar'`` even without this middleware. - AjaxCrawlMiddleware is necessary when URL doesn't contain ``'!#'``. - This is often a case for 'index' or 'main' website pages. - -AjaxCrawlMiddleware Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. setting:: AJAXCRAWL_ENABLED - -AJAXCRAWL_ENABLED -^^^^^^^^^^^^^^^^^ - -Default: ``False`` - -Whether the AjaxCrawlMiddleware will be enabled. You may want to -enable it for :ref:`broad crawls `. - -HttpProxyMiddleware settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. setting:: HTTPPROXY_ENABLED -.. setting:: HTTPPROXY_AUTH_ENCODING - -HTTPPROXY_ENABLED -^^^^^^^^^^^^^^^^^ - -Default: ``True`` - -Whether or not to enable the :class:`HttpProxyMiddleware`. - -HTTPPROXY_AUTH_ENCODING -^^^^^^^^^^^^^^^^^^^^^^^ - -Default: ``"latin-1"`` - -The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + Middleware that sets the ``User-Agent`` header. + The header value is taken from the :setting:`USER_AGENT` setting. .. _DBM: https://en.wikipedia.org/wiki/Dbm diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index a0f4b4411..30b6536c7 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -14,7 +14,7 @@ from it. If you fail to do that, and you can nonetheless access the desired data through the :ref:`DOM ` from your web browser, see -:ref:`topics-javascript-rendering`. +:ref:`topics-headless-browsing`. .. _topics-finding-data-source: @@ -83,11 +83,10 @@ request with Scrapy. It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.FormRequest`) of that request. +form parameters (see :ref:`form`) of that request. -As all major browsers allow to export the requests in `cURL -`_ format, Scrapy incorporates the method -:meth:`~scrapy.Request.from_curl()` to generate an equivalent +As all major browsers allow to export the requests in curl_ format, Scrapy +incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent :class:`~scrapy.Request` from a cURL command. To get more information visit :ref:`request from curl ` inside the network tool section. @@ -98,7 +97,7 @@ it `. You can reproduce any request with Scrapy. However, some times reproducing all necessary requests may not seem efficient in developer time. If that is your case, and crawling speed is not a major concern for you, you can alternatively -consider :ref:`JavaScript pre-rendering `. +consider :ref:`using a headless browser `. If you get the expected response `sometimes`, but not always, the issue is probably not your request, but the target server. The target server might be @@ -112,18 +111,20 @@ you may use `curl2scrapy `_. Handling different response formats =================================== +.. skip: start + Once you have a response with the desired data, how you extract the desired data from it depends on the type of response: -- If the response is HTML or XML, use :ref:`selectors +- If the response is HTML, XML or JSON, use :ref:`selectors ` as usual. -- If the response is JSON, use :func:`json.loads` to load the desired data from - :attr:`response.text `: +- If the response is JSON, use :func:`response.json() + ` to load the desired data: .. code-block:: python - data = json.loads(response.text) + data = response.json() If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a @@ -132,7 +133,7 @@ data from it depends on the type of response: .. code-block:: python - selector = Selector(data["html"]) + selector = Selector(text=data["html"]) - If the response is JavaScript, or HTML with a `` + """, + ) + r4 = HtmlResponse( + "http://www.example.com", + body=b""" + + Dummy + + + blahablsdfsal& + """, + ) + assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage") + assert get_meta_refresh(r2) == (None, None) + assert get_meta_refresh(r3) == (None, None) + assert get_meta_refresh(r4) == ( + 5.0, + "http://www.another-domain.com/base/path/target.html", + ) - def browser_open(burl): - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() - self.assertIn(b'', bbody) - return True - response = HtmlResponse(url, body=body) - assert open_in_browser(response, _openfunc=browser_open), "Browser not called" +def test_get_base_url(): + resp = HtmlResponse( + "http://www.example.com", + body=b""" + + + blahablsdfsal& + """, + ) + assert get_base_url(resp) == "http://www.example.com/img/" - resp = Response(url, body=body) - self.assertRaises(TypeError, open_in_browser, resp, debug=True) + resp2 = HtmlResponse( + "http://www.example.com", + body=b""" + blahablsdfsal&""", + ) + assert get_base_url(resp2) == "http://www.example.com" - def test_get_meta_refresh(self): - r1 = HtmlResponse( - "http://www.example.com", - body=b""" - - Dummy - blahablsdfsal& - """, - ) - r2 = HtmlResponse( - "http://www.example.com", - body=b""" - - Dummy - blahablsdfsal& - """, - ) - r3 = HtmlResponse( - "http://www.example.com", - body=b""" -