From 4c98d6068aa467f2ce88cfaf5b6ea52ae5aaf640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 25 Sep 2023 10:18:35 +0200 Subject: [PATCH 01/14] Add a template for question/help issues --- .github/ISSUE_TEMPLATE/question.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/question.md diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..63cae77e7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,13 @@ +--- +name: Question / Help +about: Ask a question about Scrapy or ask for help with your Scrapy code. +--- + +Thanks for taking an interest in Scrapy! + +The Scrapy GitHub issue tracker is not meant for questions or help. Please ask +for help in the [Scrapy community resources](https://scrapy.org/community/) +instead. + +The GitHub issue tracker's purpose is to deal with bug reports and feature +requests for the project itself. From 83fff6c9511fdde1024d8cd33d99314ddf4eb2ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20Schir=C3=A9n?= Date: Thu, 28 Sep 2023 23:50:14 +0200 Subject: [PATCH 02/14] Add dotx to exclude for documents --- scrapy/linkextractors/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 906188184..492c455fd 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -73,6 +73,7 @@ IGNORED_EXTENSIONS = [ "pps", "doc", "docx", + "dotx", "odt", "ods", "odg", From da6e75d00a8a64fea0073816d59eef2ae3b4c672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20Schir=C3=A9n?= Date: Fri, 29 Sep 2023 09:44:04 +0200 Subject: [PATCH 03/14] Added more common MS Office file extensions --- scrapy/linkextractors/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 492c455fd..6b8be909e 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -67,12 +67,21 @@ IGNORED_EXTENSIONS = [ "webm", # office suites "xls", + "xlsm", "xlsx", + "xltm", + "xltx", + "potm", + "potx", "ppt", + "pptm", "pptx", "pps", "doc", + "docb", + "docm", "docx", + "dotm", "dotx", "odt", "ods", From fe5ef0a80a0008788e35c5b9d3c016e40ddf610f Mon Sep 17 00:00:00 2001 From: Aryan <1111aryantiwari@gmail.com> Date: Sun, 1 Oct 2023 11:47:51 +0530 Subject: [PATCH 04/14] Fixed the dont_merge_cookies example in docs --- docs/topics/request-response.rst | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 41df51589..8f204fb13 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -115,21 +115,18 @@ Request objects cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. - To create a request that does not send stored cookies and does not - store received cookies, set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta `. - - Example of a request that sends manually-defined cookies and ignores - cookie storage: + Example of a request that sends manually-defined cookies: .. code-block:: python Request( url="http://www.example.com", cookies={"currency": "USD", "country": "UY"}, - meta={"dont_merge_cookies": True}, ) + It is recommended not to set the ``dont_merge_cookies`` key to ``True`` + in :attr:`request.meta ` as it causes custom cookies to be ignored. + For more info see :ref:`cookies-mw`. .. caution:: Cookies set via the ``Cookie`` header are not considered by the From 884840e3a30da24067825f97d3003d4c677b854e Mon Sep 17 00:00:00 2001 From: Andrew Armbruster Date: Sun, 1 Oct 2023 12:12:24 +0200 Subject: [PATCH 05/14] Use concurrency groups to limit duplicate jobs Leverage concurrency groups along with cancel-in-progress to favor running the most recent job. Concurrency groups are on a per workflow, per branch/tag basis. So, pushing newer updates to a branch, e.g. as part of a PR, should cancel any in progress runs of workflows that have been retriggered. See: - https://docs.github.com/en/enterprise-cloud@latest/actions/using-workflows/workflow-syntax-for-github-actions#concurrency - https://docs.github.com/en/enterprise-cloud@latest/actions/learn-github-actions/contexts#github-context --- .github/workflows/checks.yml | 4 ++++ .github/workflows/publish.yml | 4 ++++ .github/workflows/tests-macos.yml | 4 ++++ .github/workflows/tests-ubuntu.yml | 4 ++++ .github/workflows/tests-windows.yml | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ee0cb4b1e..afa713032 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,6 +1,10 @@ name: Checks on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: checks: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 22b8996b6..ec4d8fb32 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,6 +4,10 @@ on: tags: - '[0-9]+.[0-9]+.[0-9]+' +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: publish: runs-on: ubuntu-latest diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 3044a1af3..47392ff88 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,6 +1,10 @@ name: macOS on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: runs-on: macos-11 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ff92a571..84e2fdc36 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,6 +1,10 @@ name: Ubuntu on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: runs-on: ubuntu-latest diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index c8d1928d7..9949dbae8 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,6 +1,10 @@ name: Windows on: [push, pull_request] +concurrency: + group: ${{github.workflow}}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: runs-on: windows-latest From e146c3a2fc9059101f27112fc5e797f89e642fb5 Mon Sep 17 00:00:00 2001 From: Aryan <1111aryantiwari@gmail.com> Date: Mon, 2 Oct 2023 15:36:29 +0530 Subject: [PATCH 06/14] removed the entire example for dont_merge_cookies --- docs/topics/request-response.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8f204fb13..d02af2a10 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -115,15 +115,6 @@ Request objects cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. - Example of a request that sends manually-defined cookies: - - .. code-block:: python - - Request( - url="http://www.example.com", - cookies={"currency": "USD", "country": "UY"}, - ) - It is recommended not to set the ``dont_merge_cookies`` key to ``True`` in :attr:`request.meta ` as it causes custom cookies to be ignored. From ef1ed4fab7315dce1163930b16b0b898e1bbf5a6 Mon Sep 17 00:00:00 2001 From: kokobhara <146670393+kokobhara@users.noreply.github.com> Date: Mon, 2 Oct 2023 15:44:05 +0530 Subject: [PATCH 07/14] Cover PythonItemExporter backwaird-incompatible changes in 2.11 (#6081) --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c5b75aae2..fd8fa3ea3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -40,6 +40,9 @@ Backward-incompatible changes UTF-32). If you need to deal with JSON documents in an invalid encoding, use ``json.loads(response.text)`` instead. (:issue:`6016`) +- :class:`~scrapy.exporters.PythonItemExporter` used the binary output by + default but it no longer does. (:issue:`6006`, :issue:`6007`) + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ From 42adbb21043f75bec5f513eb3b7ed86c988f7562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 2 Oct 2023 14:43:30 +0200 Subject: [PATCH 08/14] Update docs/topics/request-response.rst --- docs/topics/request-response.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d02af2a10..adf3d0f4a 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -115,8 +115,9 @@ Request objects cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. - It is recommended not to set the ``dont_merge_cookies`` key to ``True`` - in :attr:`request.meta ` as it causes custom cookies to be ignored. + Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in + :attr:`request.meta ` causes custom cookies to be + ignored. For more info see :ref:`cookies-mw`. From 1ed9ed4f923ca40e0b43a72bff9e90d38db3a3bb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 2 Oct 2023 21:33:02 +0400 Subject: [PATCH 09/14] Require pyftpdlib that supports Python 3.12. --- tests/requirements.txt | 3 +-- tests/test_feedexport.py | 3 --- tests/test_pipeline_files.py | 5 ----- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 3ea7f3333..7178fb5b8 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,7 +1,6 @@ # Tests requirements attrs -# https://github.com/giampaolo/pyftpdlib/issues/560 -pyftpdlib; python_version < "3.12" +pyftpdlib >= 1.5.8 pytest pytest-cov==4.0.0 pytest-xdist diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 6b82974fa..56967c0d5 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -125,9 +125,6 @@ class FileFeedStorageTest(unittest.TestCase): path.unlink() -@pytest.mark.skipif( - sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" -) class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bf96f17b6..468751446 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,7 +1,6 @@ import dataclasses import os import random -import sys import time from datetime import datetime from io import BytesIO @@ -12,7 +11,6 @@ from unittest import mock from urllib.parse import urlparse import attr -import pytest from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest @@ -648,9 +646,6 @@ class TestGCSFilesStore(unittest.TestCase): store.bucket.get_blob.assert_called_with(expected_blob_path) -@pytest.mark.skipif( - sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" -) class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): From 60d5f391c41996bb3003f8a30c634d59f0eecf7f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 2 Oct 2023 21:35:45 +0400 Subject: [PATCH 10/14] Use the 3.12 release, enable it for all OSes. --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 20 +++++++++----------- .github/workflows/tests-windows.yml | 8 ++++---- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 47392ff88..c23b437d2 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 84e2fdc36..80b597dc2 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -21,7 +21,10 @@ jobs: - python-version: "3.11" env: TOXENV: py - - python-version: "3.11" + - python-version: "3.12" + env: + TOXENV: py + - python-version: "3.12" env: TOXENV: asyncio - python-version: pypy3.9 @@ -45,22 +48,17 @@ jobs: env: TOXENV: botocore-pinned - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: extra-deps - - python-version: "3.11" + - python-version: "3.12" env: TOXENV: botocore - - python-version: "3.12.0-rc.2" - env: - TOXENV: py - - python-version: "3.12.0-rc.2" + # keep until uvloop supports 3.12 + - python-version: "3.11" env: TOXENV: asyncio - - python-version: "3.12.0-rc.2" - env: - TOXENV: extra-deps steps: - uses: actions/checkout@v3 @@ -71,7 +69,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || contains(matrix.python-version, '3.12.0') + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 9949dbae8..955924349 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -21,13 +21,13 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.10" - env: - TOXENV: asyncio - python-version: "3.11" env: TOXENV: py - - python-version: "3.11" + - python-version: "3.12" + env: + TOXENV: py + - python-version: "3.12" env: TOXENV: asyncio From 9ae8d97d81fd0b660603473a3d0601f9d1aa77f5 Mon Sep 17 00:00:00 2001 From: Klaus Rettinghaus Date: Tue, 3 Oct 2023 13:04:18 +0200 Subject: [PATCH 11/14] Update GitHub Actions checkout action to version 4 (#6084) --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 2 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- .github/workflows/tests-windows.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index afa713032..0ceb53848 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -26,7 +26,7 @@ jobs: TOXENV: twinecheck steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -42,5 +42,5 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ec4d8fb32..dd778fc64 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: publish: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: 3.11 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 47392ff88..d0d7c248a 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -14,7 +14,7 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 84e2fdc36..b6d5cd27a 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -63,7 +63,7 @@ jobs: TOXENV: extra-deps steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 9949dbae8..eaf4df41b 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -32,7 +32,7 @@ jobs: TOXENV: asyncio steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 From fb4debda04cff4e33daf62d95c71ab2919c37c07 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Oct 2023 16:47:03 +0400 Subject: [PATCH 12/14] Use path_to_file_uri to fix the test on Windows. --- tests/test_downloader_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 57211d97a..f12243e1d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -127,7 +127,7 @@ class FileTestCase(unittest.TestCase): return self.download_request(request, Spider("foo")).addCallback(_test) def test_non_existent(self): - request = Request(f"file://{self.mktemp()}") + request = Request(path_to_file_uri(self.mktemp())) d = self.download_request(request, Spider("foo")) return self.assertFailure(d, OSError) From a338873e3acbb6024de2c371392dc5beaa73376d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 3 Oct 2023 16:50:52 +0400 Subject: [PATCH 13/14] Re-enable bpython on Python 3.12. --- tests/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 7178fb5b8..c07fda2d6 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,8 +9,7 @@ testfixtures # uvloop currently doesn't build on 3.12 uvloop; platform_system != "Windows" and python_version < "3.12" -# bpython requires greenlet which currently doesn't build on 3.12 -bpython; python_version < "3.12" # optional for shell wrapper tests +bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests # 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests From c65567988da2f6dd8ad894cf0cf57f2c074be10f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 6 Oct 2023 13:27:02 +0400 Subject: [PATCH 14/14] Change supported PyPy versions to 3.9 and 3.10. (#6087) * Change support PyPy versions to 3.9 and 3.10. * Update the RTD URL for coverage. * Move pypy3-pinned back to pypy3.8. --- .github/workflows/tests-ubuntu.yml | 5 ++++- docs/conf.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 7ac0305f5..a307eb337 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -30,6 +30,9 @@ jobs: - python-version: pypy3.9 env: TOXENV: pypy3 + - python-version: pypy3.10 + env: + TOXENV: pypy3 # pinned deps - python-version: 3.8.17 @@ -69,7 +72,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') + if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/docs/conf.py b/docs/conf.py index 38ca81932..9ca0f817a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -276,7 +276,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/stable", None), + "coverage": ("https://coverage.readthedocs.io/en/latest", None), "cryptography": ("https://cryptography.io/en/latest/", None), "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),