Merge branch 'master' into add-black-formatter

docs/news.rst

@@ -12,7 +12,7 @@ New features
 ~~~~~~~~~~~~
 
 -   Relaxed the restriction introduced in 2.6.2 so that the
-    ``Proxy-Authentication`` header can again be set explicitly, as long as the
+    ``Proxy-Authorization`` header can again be set explicitly, as long as the
     proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and
     for as long as that proxy URL remains the same; this restores compatibility
     with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`).
@@ -281,7 +281,7 @@ Scrapy 2.6.2 (2022-07-25)
     processes a request with :reqmeta:`proxy` metadata, and that
     :reqmeta:`proxy` metadata includes proxy credentials,
     :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
-    the ``Proxy-Authentication`` header, but only if that header is not already
+    the ``Proxy-Authorization`` header, but only if that header is not already
     set.
 
     There are third-party proxy-rotation downloader middlewares that set
@@ -294,7 +294,7 @@ Scrapy 2.6.2 (2022-07-25)
 
     These third-party proxy-rotation downloader middlewares could change the
     :reqmeta:`proxy` metadata of a request to a new value, but fail to remove
-    the ``Proxy-Authentication`` header from the previous value of the
+    the ``Proxy-Authorization`` header from the previous value of the
     :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
     to a different proxy.
 
@@ -2250,7 +2250,7 @@ Scrapy 1.8.3 (2022-07-25)
     processes a request with :reqmeta:`proxy` metadata, and that
     :reqmeta:`proxy` metadata includes proxy credentials,
     :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
-    the ``Proxy-Authentication`` header, but only if that header is not already
+    the ``Proxy-Authorization`` header, but only if that header is not already
     set.
 
     There are third-party proxy-rotation downloader middlewares that set
@@ -2263,7 +2263,7 @@ Scrapy 1.8.3 (2022-07-25)
 
     These third-party proxy-rotation downloader middlewares could change the
     :reqmeta:`proxy` metadata of a request to a new value, but fail to remove
-    the ``Proxy-Authentication`` header from the previous value of the
+    the ``Proxy-Authorization`` header from the previous value of the
     :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
     to a different proxy.
 

docs/topics/settings.rst

@@ -652,8 +652,9 @@ per ip address instead of per domain.
 
 .. _spider-download_delay-attribute:
 
-You can also change this setting per spider by setting ``download_delay``
-spider attribute.
+.. note::
+
+    This delay can be set per spider using :attr:`download_delay` spider attribute.
 
 .. setting:: DOWNLOAD_HANDLERS
 

scrapy/pipelines/images.py

@@ -181,8 +181,8 @@ class ImagesPipeline(FilesPipeline):
                 stacklevel=2,
             )
 
-        if image.format == "PNG" and image.mode == "RGBA":
-            background = self._Image.new("RGBA", image.size, (255, 255, 255))
+        if image.format in ('PNG', 'WEBP') and image.mode == 'RGBA':
+            background = self._Image.new('RGBA', image.size, (255, 255, 255))
             background.paste(image, image)
             image = background.convert("RGB")
         elif image.mode == "P":

scrapy/shell.py

@@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
+from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
 
 
 class Shell:
@@ -77,6 +78,10 @@ class Shell:
             )
 
     def _schedule(self, request, spider):
+        if is_asyncio_reactor_installed():
+            # set the asyncio event loop for the current thread
+            event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP']
+            set_asyncio_event_loop(event_loop_path)
         spider = self._open_spider(request, spider)
         d = _request_deferred(request)
         d.addCallback(lambda x: (x, spider))

scrapy/utils/log.py

@@ -46,6 +46,9 @@ DEFAULT_LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
     "loggers": {
+        "filelock": {
+            "level": "ERROR",
+        },
         "hpack": {
             "level": "ERROR",
         },

scrapy/utils/reactor.py

@@ -73,14 +73,7 @@ def install_reactor(reactor_path, event_loop_path=None):
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
         with suppress(error.ReactorAlreadyInstalledError):
-            policy = get_asyncio_event_loop_policy()
-            if event_loop_path is not None:
-                event_loop_class = load_object(event_loop_path)
-                event_loop = event_loop_class()
-                asyncio.set_event_loop(event_loop)
-            else:
-                event_loop = policy.get_event_loop()
-
+            event_loop = set_asyncio_event_loop(event_loop_path)
             asyncioreactor.install(eventloop=event_loop)
     else:
         *module, _ = reactor_path.split(".")
@@ -90,6 +83,25 @@ def install_reactor(reactor_path, event_loop_path=None):
             installer()
 
 
+def set_asyncio_event_loop(event_loop_path):
+    """Sets and returns the event loop with specified import path."""
+    policy = get_asyncio_event_loop_policy()
+    if event_loop_path is not None:
+        event_loop_class = load_object(event_loop_path)
+        event_loop = event_loop_class()
+        asyncio.set_event_loop(event_loop)
+    else:
+        try:
+            event_loop = policy.get_event_loop()
+        except RuntimeError:
+            # `get_event_loop` is expected to fail when called from a new thread
+            # with no asyncio event loop yet installed. Such is the case when
+            # called from `scrapy shell`
+            event_loop = policy.new_event_loop()
+            asyncio.set_event_loop(event_loop)
+    return event_loop
+
+
 def verify_installed_reactor(reactor_path):
     """Raises :exc:`Exception` if the installed
     :mod:`~twisted.internet.reactor` does not match the specified import

setup.py

@@ -59,13 +59,14 @@ setup(
         "Source": "https://github.com/scrapy/scrapy",
         "Tracker": "https://github.com/scrapy/scrapy/issues",
     },
-    description="A high-level Web Crawling and Web Scraping framework",
-    long_description=open("README.rst", encoding="utf-8").read(),
-    author="Scrapy developers",
-    maintainer="Pablo Hoffman",
-    maintainer_email="pablo@pablohoffman.com",
-    license="BSD",
-    packages=find_packages(exclude=("tests", "tests.*")),
+    description='A high-level Web Crawling and Web Scraping framework',
+    long_description=open('README.rst', encoding="utf-8").read(),
+    author='Scrapy developers',
+    author_email='pablo@pablohoffman.com',
+    maintainer='Pablo Hoffman',
+    maintainer_email='pablo@pablohoffman.com',
+    license='BSD',
+    packages=find_packages(exclude=('tests', 'tests.*')),
     include_package_data=True,
     zip_safe=False,
     entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]},

tests/test_command_shell.py

@@ -125,4 +125,15 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
         url = "www.somedomainthatdoesntexi.st"
         errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False)
         self.assertEqual(errcode, 1, out or err)
-        self.assertIn(b"DNS lookup failed", err)
+        self.assertIn(b'DNS lookup failed', err)
+
+    @defer.inlineCallbacks
+    def test_shell_fetch_async(self):
+        reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+        url = self.url('/html')
+        code = f"fetch('{url}')"
+        args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"]
+        _, _, err = yield self.execute(args, check_code=True)
+        self.assertNotIn(
+            b"RuntimeError: There is no current event loop in thread", err
+        )

tests/test_http_response.py

@@ -2,20 +2,17 @@ import codecs
 import unittest
 from unittest import mock
 
+from packaging.version import Version as parse_version
+from pytest import mark
+from w3lib import __version__ as w3lib_version
 from w3lib.encoding import resolve_encoding
 
-from scrapy.http import (
-    Request,
-    Response,
-    TextResponse,
-    HtmlResponse,
-    XmlResponse,
-    Headers,
-)
+from scrapy.exceptions import NotSupported
+from scrapy.http import (Headers, HtmlResponse, Request, Response,
+                         TextResponse, XmlResponse)
+from scrapy.link import Link
 from scrapy.selector import Selector
 from scrapy.utils.python import to_unicode
-from scrapy.exceptions import NotSupported
-from scrapy.link import Link
 from tests import get_testdata
 
 
@@ -209,13 +206,23 @@ class BaseResponseTest(unittest.TestCase):
         r = self.response_class("http://example.com")
         self.assertRaises(ValueError, r.follow, None)
 
+    @mark.xfail(
+        parse_version(w3lib_version) < parse_version("2.1.1"),
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
     def test_follow_whitespace_url(self):
-        self._assert_followed_url("foo ", "http://example.com/foo%20")
+        self._assert_followed_url('foo ',
+                                  'http://example.com/foo')
 
+    @mark.xfail(
+        parse_version(w3lib_version) < parse_version("2.1.1"),
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
     def test_follow_whitespace_link(self):
-        self._assert_followed_url(
-            Link("http://example.com/foo "), "http://example.com/foo%20"
-        )
+        self._assert_followed_url(Link('http://example.com/foo '),
+                                  'http://example.com/foo')
 
     def test_follow_flags(self):
         res = self.response_class("http://example.com/")