Merge remote-tracking branch 'scrapy/2.11' into 2.11-redos

2023-12-15 11:52:35 +01:00 · 2023-12-15 11:52:35 +01:00 · cd6846b763
parent b0dbd0e9af 49b284ab85
commit cd6846b763
23 changed files with 162 additions and 87 deletions
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@ -8,7 +8,7 @@ jobs:
      fail-fast: false
      matrix:
        include:
-        - python-version: "3.11"
+        - python-version: "3.12"
          env:
            TOXENV: pylint
        - python-version: 3.8
@ -17,7 +17,7 @@ jobs:
        - python-version: "3.11"  # Keep in sync with .readthedocs.yml
          env:
            TOXENV: docs
-        - python-version: "3.11"
+        - python-version: "3.12"
          env:
            TOXENV: twinecheck

--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@ -11,7 +11,7 @@ jobs:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
-          python-version: 3.11
+          python-version: 3.12
      - run: | 
          pip install --upgrade build twine
          python -m build
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@ -7,7 +7,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]

    steps:
    - uses: actions/checkout@v3
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@ -17,7 +17,10 @@ jobs:
        - python-version: "3.11"
          env:
            TOXENV: py
-        - python-version: "3.11"
+        - python-version: "3.12"
+          env:
+            TOXENV: py
+        - python-version: "3.12"
          env:
            TOXENV: asyncio
        - python-version: pypy3.9
@ -41,29 +44,15 @@ jobs:
          env:
            TOXENV: botocore-pinned

-        - python-version: "3.11"
+        - python-version: "3.12"
          env:
            TOXENV: extra-deps
-        - python-version: "3.11"
+        - python-version: "3.12"
          env:
            TOXENV: botocore
-        - python-version: "3.11"
+        - python-version: "3.12"
          env:
            TOXENV: slow
-
-        - python-version: "3.12.0-rc.2"
-          env:
-            TOXENV: py
-        - python-version: "3.12.0-rc.2"
-          env:
-            TOXENV: asyncio
-        - python-version: "3.12.0-rc.2"
-          env:
-            TOXENV: extra-deps
-        - python-version: "3.12.0-rc.2"
-          env:
-            TOXENV: slow
-
    steps:
    - uses: actions/checkout@v3

@ -73,7 +62,7 @@ jobs:
        python-version: ${{ matrix.python-version }}

    - name: Install system libraries
-      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')  || contains(matrix.python-version, '3.12.0')
+      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned')
      run: |
        sudo apt-get update
        sudo apt-get install libxml2-dev libxslt-dev
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@ -17,13 +17,13 @@ jobs:
        - python-version: "3.10"
          env:
            TOXENV: py
-        - python-version: "3.10"
-          env:
-            TOXENV: asyncio
        - python-version: "3.11"
          env:
            TOXENV: py
-        - python-version: "3.11"
+        - python-version: "3.12"
+          env:
+            TOXENV: py
+        - python-version: "3.12"
          env:
            TOXENV: asyncio

--- a/README.rst
+++ b/README.rst
@ -17,9 +17,10 @@ Scrapy
   :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
   :alt: Ubuntu

-.. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
-   :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
-   :alt: macOS
+.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
+   .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
+   .. :alt: macOS
+

 .. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
   :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
@ -41,7 +42,7 @@ Scrapy
 Overview
 ========

-Scrapy is a fast high-level web crawling and web scraping framework, used to
+Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to
 crawl websites and extract structured data from their pages. It can be used for
 a wide range of purposes, from data mining to monitoring and automated testing.

@ -110,4 +111,4 @@ See https://scrapy.org/companies/ for a list.
 Commercial Support
 ==================

-See https://scrapy.org/support/ for details.
+See https://scrapy.org/support/ for details.
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@ -493,7 +493,15 @@ in the callback, as you can see below:
                    "tags": quote.css("div.tags a.tag::text").getall(),
                }

-If you run this spider, it will output the extracted data with the log::
+To run this spider, exit the scrapy shell by entering::
+
+    quit()
+
+Then, run::
+
+   scrapy crawl quotes
+
+Now, it should output the extracted data with the log::

    2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/>
    {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'}
--- a/docs/news.rst
+++ b/docs/news.rst
@ -59,8 +59,10 @@ Backward-incompatible changes
    :meth:`scrapy.crawler.Crawler.__init__` and before the settings are
    finalized and frozen. This change was needed to allow changing the settings
    in :meth:`scrapy.Spider.from_crawler`. If you want to access the final
-    setting values in the spider code as early as possible you can do this in
-    :meth:`~scrapy.Spider.start_requests`. (:issue:`6038`)
+    setting values and the initialized :class:`~scrapy.crawler.Crawler`
+    attributes in the spider code as early as possible you can do this in
+    :meth:`~scrapy.Spider.start_requests` or in a handler of the
+    :signal:`engine_started` signal. (:issue:`6038`)

 -   The :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method now
    requires the response to be in a valid JSON encoding (UTF-8, UTF-16, or
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@ -115,20 +115,9 @@ Request objects
        cookies for that domain and will be sent again in future requests.
        That's the typical behaviour of any regular web browser.

-        To create a request that does not send stored cookies and does not
-        store received cookies, set the ``dont_merge_cookies`` key to ``True``
-        in :attr:`request.meta <scrapy.Request.meta>`.
-
-        Example of a request that sends manually-defined cookies and ignores
-        cookie storage:
-
-        .. code-block:: python
-
-            Request(
-                url="http://www.example.com",
-                cookies={"currency": "USD", "country": "UY"},
-                meta={"dont_merge_cookies": True},
-            )
+        Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in
+        :attr:`request.meta <scrapy.Request.meta>` causes custom cookies to be
+        ignored.

        For more info see :ref:`cookies-mw`.

--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@ -142,8 +142,14 @@ scrapy.Spider
           method, which is handy if you want to modify them based on
           arguments. As a consequence, these settings aren't the final values
           as they can be modified later by e.g. :ref:`add-ons
-           <topics-addons>`. The final settings are available in the
-           :meth:`start_requests` method and later.
+           <topics-addons>`. For the same reason, most of the
+           :class:`~scrapy.crawler.Crawler` attributes aren't initialized at
+           this point.
+
+           The final settings and the initialized
+           :class:`~scrapy.crawler.Crawler` attributes are available in the
+           :meth:`start_requests` method, handlers of the
+           :signal:`engine_started` signal and later.

       :param crawler: crawler to which the spider will be bound
       :type crawler: :class:`~scrapy.crawler.Crawler` instance
--- a/scrapy/contracts/init.py
+++ b/scrapy/contracts/init.py
@ -41,7 +41,9 @@ class Contract:
                    cb_result = cb(response, **cb_kwargs)
                    if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
                        raise TypeError("Contracts don't support async callbacks")
-                    return list(iterate_spider_output(cb_result))
+                    return list(  # pylint: disable=return-in-finally
+                        iterate_spider_output(cb_result)
+                    )

            request.callback = wrapper

@ -68,7 +70,7 @@ class Contract:
                else:
                    results.addSuccess(self.testcase_post)
                finally:
-                    return output
+                    return output  # pylint: disable=return-in-finally

            request.callback = wrapper

--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@ -404,8 +404,8 @@ class CrawlerProcess(CrawlerRunner):
        :param bool stop_after_crawl: stop or not the reactor when all
            crawlers have finished

-        :param bool install_signal_handlers: whether to install the shutdown
-            handlers (default: True)
+        :param bool install_signal_handlers: whether to install the OS signal
+            handlers from Twisted and Scrapy (default: True)
        """
        from twisted.internet import reactor

@ -416,15 +416,17 @@ class CrawlerProcess(CrawlerRunner):
                return
            d.addBoth(self._stop_reactor)

-        if install_signal_handlers:
-            install_shutdown_handlers(self._signal_shutdown)
        resolver_class = load_object(self.settings["DNS_RESOLVER"])
        resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
        resolver.install_on_reactor()
        tp = reactor.getThreadPool()
        tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
        reactor.addSystemEventTrigger("before", "shutdown", self.stop)
-        reactor.run(installSignalHandlers=False)  # blocking call
+        if install_signal_handlers:
+            reactor.addSystemEventTrigger(
+                "after", "startup", install_shutdown_handlers, self._signal_shutdown
+            )
+        reactor.run(installSignalHandlers=install_signal_handlers)  # blocking call

    def _graceful_stop_reactor(self) -> Deferred:
        d = self.stop()
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@ -19,13 +19,10 @@ def install_shutdown_handlers(
    function: SignalHandlerT, override_sigint: bool = True
 ) -> None:
    """Install the given function as a signal handler for all common shutdown
-    signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the
-    SIGINT handler won't be install if there is already a handler in place
-    (e.g.  Pdb)
+    signals (such as SIGINT, SIGTERM, etc). If ``override_sigint`` is ``False`` the
+    SIGINT handler won't be installed if there is already a handler in place
+    (e.g. Pdb)
    """
-    from twisted.internet import reactor
-
-    reactor._handleSignals()
    signal.signal(signal.SIGTERM, function)
    if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
        signal.signal(signal.SIGINT, function)
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@ -2,7 +2,7 @@ from __future__ import annotations

 import os
 import sys
-from typing import Iterable, Optional, Tuple, cast
+from typing import Iterable, List, Optional, Tuple, cast

 from twisted.internet.defer import Deferred
 from twisted.internet.error import ProcessTerminated
@ -26,14 +26,15 @@ class ProcessTest:
        env = os.environ.copy()
        if settings is not None:
            env["SCRAPY_SETTINGS_MODULE"] = settings
+        assert self.command
        cmd = self.prefix + [self.command] + list(args)
        pp = TestProcessProtocol()
-        pp.deferred.addBoth(self._process_finished, cmd, check_code)
+        pp.deferred.addCallback(self._process_finished, cmd, check_code)
        reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd)
        return pp.deferred

    def _process_finished(
-        self, pp: TestProcessProtocol, cmd: str, check_code: bool
+        self, pp: TestProcessProtocol, cmd: List[str], check_code: bool
    ) -> Tuple[int, bytes, bytes]:
        if pp.exitcode and check_code:
            msg = f"process {cmd} exit with code {pp.exitcode}"
--- a/setup.py
+++ b/setup.py
@ -6,8 +6,7 @@ version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip()


 install_requires = [
-    # 23.8.0 incompatibility: https://github.com/scrapy/scrapy/issues/6024
-    "Twisted>=18.9.0,<23.8.0",
+    "Twisted>=18.9.0",
    "cryptography>=36.0.0",
    "cssselect>=0.9.1",
    "itemloaders>=1.0.1",
--- a/tests/CrawlerProcess/sleeping.py
+++ b/tests/CrawlerProcess/sleeping.py
@ -0,0 +1,24 @@
+from twisted.internet.defer import Deferred
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from scrapy.utils.defer import maybe_deferred_to_future
+
+
+class SleepingSpider(scrapy.Spider):
+    name = "sleeping"
+
+    start_urls = ["data:,;"]
+
+    async def parse(self, response):
+        from twisted.internet import reactor
+
+        d = Deferred()
+        reactor.callLater(int(self.sleep), d.callback, None)
+        await maybe_deferred_to_future(d)
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(SleepingSpider)
+process.start()
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@ -1,17 +1,15 @@
 # Tests requirements
 attrs
-# https://github.com/giampaolo/pyftpdlib/issues/560
-pyftpdlib; python_version < "3.12"
+pexpect >= 4.8.0
+pyftpdlib >= 1.5.8
 pytest
 pytest-cov==4.0.0
 pytest-xdist
 sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures
-# uvloop currently doesn't build on 3.12
-uvloop; platform_system != "Windows" and python_version < "3.12"
+uvloop; platform_system != "Windows"

-# bpython requires greenlet which currently doesn't build on 3.12
-bpython; python_version < "3.12"  # optional for shell wrapper tests
+bpython  # optional for shell wrapper tests
 brotli; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 # 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072
 brotli==1.0.9; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@ -1,11 +1,16 @@
+import os
+import sys
+from io import BytesIO
 from pathlib import Path

+from pexpect.popen_spawn import PopenSpawn
 from twisted.internet import defer
 from twisted.trial import unittest

 from scrapy.utils.testproc import ProcessTest
 from scrapy.utils.testsite import SiteTest
 from tests import NON_EXISTING_RESOLVABLE, tests_datadir
+from tests.mockserver import MockServer


 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@ -133,3 +138,27 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
        args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"]
        _, _, err = yield self.execute(args, check_code=True)
        self.assertNotIn(b"RuntimeError: There is no current event loop in thread", err)
+
+
+class InteractiveShellTest(unittest.TestCase):
+    def test_fetch(self):
+        args = (
+            sys.executable,
+            "-m",
+            "scrapy.cmdline",
+            "shell",
+        )
+        env = os.environ.copy()
+        env["SCRAPY_PYTHON_SHELL"] = "python"
+        logfile = BytesIO()
+        p = PopenSpawn(args, env=env, timeout=5)
+        p.logfile_read = logfile
+        p.expect_exact("Available Scrapy objects")
+        with MockServer() as mockserver:
+            p.sendline(f"fetch('{mockserver.url('/')}')")
+            p.sendline("type(response)")
+            p.expect_exact("HtmlResponse")
+        p.sendeof()
+        p.wait()
+        logfile.seek(0)
+        self.assertNotIn("Traceback", logfile.read().decode())
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@ -1,13 +1,16 @@
 import logging
 import os
 import platform
+import signal
 import subprocess
 import sys
 import warnings
 from pathlib import Path
+from typing import List

 import pytest
 from packaging.version import parse as parse_version
+from pexpect.popen_spawn import PopenSpawn
 from pytest import mark, raises
 from twisted.internet import defer
 from twisted.trial import unittest
@ -289,9 +292,12 @@ class ScriptRunnerMixin:
    script_dir: Path
    cwd = os.getcwd()

-    def run_script(self, script_name: str, *script_args):
+    def get_script_args(self, script_name: str, *script_args: str) -> List[str]:
        script_path = self.script_dir / script_name
-        args = [sys.executable, str(script_path)] + list(script_args)
+        return [sys.executable, str(script_path)] + list(script_args)
+
+    def run_script(self, script_name: str, *script_args: str) -> str:
+        args = self.get_script_args(script_name, *script_args)
        p = subprocess.Popen(
            args,
            env=get_mockserver_env(),
@ -517,6 +523,36 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
        self.assertIn("Spider closed (finished)", log)
        self.assertIn("The value of FOO is 42", log)

+    def test_shutdown_graceful(self):
+        sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
+        args = self.get_script_args("sleeping.py", "-a", "sleep=3")
+        p = PopenSpawn(args, timeout=5)
+        p.expect_exact("Spider opened")
+        p.expect_exact("Crawled (200)")
+        p.kill(sig)
+        p.expect_exact("shutting down gracefully")
+        p.expect_exact("Spider closed (shutdown)")
+        p.wait()
+
+    @defer.inlineCallbacks
+    def test_shutdown_forced(self):
+        from twisted.internet import reactor
+
+        sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
+        args = self.get_script_args("sleeping.py", "-a", "sleep=10")
+        p = PopenSpawn(args, timeout=5)
+        p.expect_exact("Spider opened")
+        p.expect_exact("Crawled (200)")
+        p.kill(sig)
+        p.expect_exact("shutting down gracefully")
+        # sending the second signal too fast often causes problems
+        d = defer.Deferred()
+        reactor.callLater(0.1, d.callback, None)
+        yield d
+        p.kill(sig)
+        p.expect_exact("forcing unclean shutdown")
+        p.wait()
+

 class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
    script_dir = Path(__file__).parent.resolve() / "CrawlerRunner"
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@ -127,7 +127,7 @@ class FileTestCase(unittest.TestCase):
        return self.download_request(request, Spider("foo")).addCallback(_test)

    def test_non_existent(self):
-        request = Request(f"file://{self.mktemp()}")
+        request = Request(path_to_file_uri(self.mktemp()))
        d = self.download_request(request, Spider("foo"))
        return self.assertFailure(d, OSError)

--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@ -125,9 +125,6 @@ class FileFeedStorageTest(unittest.TestCase):
            path.unlink()


-@pytest.mark.skipif(
-    sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet"
-)
 class FTPFeedStorageTest(unittest.TestCase):
    def get_test_spider(self, settings=None):
        class TestSpider(scrapy.Spider):
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@ -1,7 +1,6 @@
 import dataclasses
 import os
 import random
-import sys
 import time
 from datetime import datetime
 from io import BytesIO
@ -12,7 +11,6 @@ from unittest import mock
 from urllib.parse import urlparse

 import attr
-import pytest
 from itemadapter import ItemAdapter
 from twisted.internet import defer
 from twisted.trial import unittest
@ -648,9 +646,6 @@ class TestGCSFilesStore(unittest.TestCase):
                    store.bucket.get_blob.assert_called_with(expected_blob_path)


-@pytest.mark.skipif(
-    sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet"
-)
 class TestFTPFileStore(unittest.TestCase):
    @defer.inlineCallbacks
    def test_persist(self):
--- a/tox.ini
+++ b/tox.ini
@ -57,7 +57,7 @@ commands =
 basepython = python3
 deps =
    {[testenv:extra-deps]deps}
-    pylint==2.17.5
+    pylint==3.0.1
 commands =
    pylint conftest.py docs extras scrapy setup.py tests