Merge remote-tracking branch 'origin/master' into signals-proper

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,13 @@
+---
+name: Question / Help
+about: Ask a question about Scrapy or ask for help with your Scrapy code.
+---
+
+Thanks for taking an interest in Scrapy!
+
+The Scrapy GitHub issue tracker is not meant for questions or help. Please ask
+for help in the [Scrapy community resources](https://scrapy.org/community/)
+instead.
+
+The GitHub issue tracker's purpose is to deal with bug reports and feature
+requests for the project itself.

.github/workflows/checks.yml

@@ -1,6 +1,10 @@
 name: Checks
 on: [push, pull_request]
 
+concurrency:
+  group: ${{github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   checks:
     runs-on: ubuntu-latest
@@ -22,7 +26,7 @@ jobs:
             TOXENV: twinecheck
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
@@ -38,5 +42,5 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: pre-commit/action@v3.0.0

.github/workflows/publish.yml

@@ -4,11 +4,15 @@ on:
     tags:
       - '[0-9]+.[0-9]+.[0-9]+'
 
+concurrency:
+  group: ${{github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: 3.11

.github/workflows/tests-macos.yml

@@ -1,16 +1,20 @@
 name: macOS
 on: [push, pull_request]
 
+concurrency:
+  group: ${{github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   tests:
     runs-on: macos-11
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4

.github/workflows/tests-ubuntu.yml

@@ -1,6 +1,10 @@
 name: Ubuntu
 on: [push, pull_request]
 
+concurrency:
+  group: ${{github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   tests:
     runs-on: ubuntu-latest
@@ -17,12 +21,18 @@ jobs:
         - python-version: "3.11"
           env:
             TOXENV: py
-        - python-version: "3.11"
+        - python-version: "3.12"
+          env:
+            TOXENV: py
+        - python-version: "3.12"
           env:
             TOXENV: asyncio
         - python-version: pypy3.9
           env:
             TOXENV: pypy3
+        - python-version: pypy3.10
+          env:
+            TOXENV: pypy3
 
         # pinned deps
         - python-version: 3.8.17
@@ -41,25 +51,20 @@ jobs:
           env:
             TOXENV: botocore-pinned
 
-        - python-version: "3.11"
+        - python-version: "3.12"
           env:
             TOXENV: extra-deps
-        - python-version: "3.11"
+        - python-version: "3.12"
           env:
             TOXENV: botocore
 
-        - python-version: "3.12.0-rc.2"
-          env:
-            TOXENV: py
-        - python-version: "3.12.0-rc.2"
+        # keep until uvloop supports 3.12
+        - python-version: "3.11"
           env:
             TOXENV: asyncio
-        - python-version: "3.12.0-rc.2"
-          env:
-            TOXENV: extra-deps
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
@@ -67,7 +72,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install system libraries
-      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')  || contains(matrix.python-version, '3.12.0')
+      if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned')
       run: |
         sudo apt-get update
         sudo apt-get install libxml2-dev libxslt-dev

.github/workflows/tests-windows.yml

@@ -1,6 +1,10 @@
 name: Windows
 on: [push, pull_request]
 
+concurrency:
+  group: ${{github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   tests:
     runs-on: windows-latest
@@ -17,18 +21,18 @@ jobs:
         - python-version: "3.10"
           env:
             TOXENV: py
-        - python-version: "3.10"
-          env:
-            TOXENV: asyncio
         - python-version: "3.11"
           env:
             TOXENV: py
-        - python-version: "3.11"
+        - python-version: "3.12"
+          env:
+            TOXENV: py
+        - python-version: "3.12"
           env:
             TOXENV: asyncio
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4

docs/conf.py

@@ -276,7 +276,7 @@ coverage_ignore_pyobjects = [
 
 intersphinx_mapping = {
     "attrs": ("https://www.attrs.org/en/stable/", None),
-    "coverage": ("https://coverage.readthedocs.io/en/stable", None),
+    "coverage": ("https://coverage.readthedocs.io/en/latest", None),
     "cryptography": ("https://cryptography.io/en/latest/", None),
     "cssselect": ("https://cssselect.readthedocs.io/en/latest", None),
     "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),

docs/news.rst

@@ -40,6 +40,9 @@ Backward-incompatible changes
     UTF-32). If you need to deal with JSON documents in an invalid encoding,
     use ``json.loads(response.text)`` instead. (:issue:`6016`)
 
+-   :class:`~scrapy.exporters.PythonItemExporter` used the binary output by
+    default but it no longer does. (:issue:`6006`, :issue:`6007`)
+
 Deprecation removals
 ~~~~~~~~~~~~~~~~~~~~
 

docs/topics/request-response.rst

@@ -115,20 +115,9 @@ Request objects
         cookies for that domain and will be sent again in future requests.
         That's the typical behaviour of any regular web browser.
 
-        To create a request that does not send stored cookies and does not
-        store received cookies, set the ``dont_merge_cookies`` key to ``True``
-        in :attr:`request.meta <scrapy.Request.meta>`.
-
-        Example of a request that sends manually-defined cookies and ignores
-        cookie storage:
-
-        .. code-block:: python
-
-            Request(
-                url="http://www.example.com",
-                cookies={"currency": "USD", "country": "UY"},
-                meta={"dont_merge_cookies": True},
-            )
+        Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in
+        :attr:`request.meta <scrapy.Request.meta>` causes custom cookies to be
+        ignored.
 
         For more info see :ref:`cookies-mw`.
 

scrapy/linkextractors/__init__.py

@@ -67,12 +67,22 @@ IGNORED_EXTENSIONS = [
     "webm",
     # office suites
     "xls",
+    "xlsm",
     "xlsx",
+    "xltm",
+    "xltx",
+    "potm",
+    "potx",
     "ppt",
+    "pptm",
     "pptx",
     "pps",
     "doc",
+    "docb",
+    "docm",
     "docx",
+    "dotm",
+    "dotx",
     "odt",
     "ods",
     "odg",

tests/requirements.txt

@@ -1,8 +1,7 @@
 # Tests requirements
 attrs
 pexpect >= 4.8.0
-# https://github.com/giampaolo/pyftpdlib/issues/560
-pyftpdlib; python_version < "3.12"
+pyftpdlib >= 1.5.8
 pytest
 pytest-cov==4.0.0
 pytest-xdist
@@ -11,8 +10,7 @@ testfixtures
 # uvloop currently doesn't build on 3.12
 uvloop; platform_system != "Windows" and python_version < "3.12"
 
-# bpython requires greenlet which currently doesn't build on 3.12
-bpython; python_version < "3.12"  # optional for shell wrapper tests
+bpython  # optional for shell wrapper tests
 brotli; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 # 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072
 brotli==1.0.9; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests

tests/test_downloader_handlers.py

@@ -127,7 +127,7 @@ class FileTestCase(unittest.TestCase):
         return self.download_request(request, Spider("foo")).addCallback(_test)
 
     def test_non_existent(self):
-        request = Request(f"file://{self.mktemp()}")
+        request = Request(path_to_file_uri(self.mktemp()))
         d = self.download_request(request, Spider("foo"))
         return self.assertFailure(d, OSError)
 

tests/test_feedexport.py

@@ -125,9 +125,6 @@ class FileFeedStorageTest(unittest.TestCase):
             path.unlink()
 
 
-@pytest.mark.skipif(
-    sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet"
-)
 class FTPFeedStorageTest(unittest.TestCase):
     def get_test_spider(self, settings=None):
         class TestSpider(scrapy.Spider):

tests/test_pipeline_files.py

@@ -1,7 +1,6 @@
 import dataclasses
 import os
 import random
-import sys
 import time
 from datetime import datetime
 from io import BytesIO
@@ -12,7 +11,6 @@ from unittest import mock
 from urllib.parse import urlparse
 
 import attr
-import pytest
 from itemadapter import ItemAdapter
 from twisted.internet import defer
 from twisted.trial import unittest
@@ -648,9 +646,6 @@ class TestGCSFilesStore(unittest.TestCase):
                     store.bucket.get_blob.assert_called_with(expected_blob_path)
 
 
-@pytest.mark.skipif(
-    sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet"
-)
 class TestFTPFileStore(unittest.TestCase):
     @defer.inlineCallbacks
     def test_persist(self):