diff --git a/docs/news.rst b/docs/news.rst index e5fc2971a..c97de0ed8 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,7 +12,7 @@ New features ~~~~~~~~~~~~ - Relaxed the restriction introduced in 2.6.2 so that the - ``Proxy-Authentication`` header can again be set explicitly, as long as the + ``Proxy-Authorization`` header can again be set explicitly, as long as the proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and for as long as that proxy URL remains the same; this restores compatibility with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`). @@ -281,7 +281,7 @@ Scrapy 2.6.2 (2022-07-25) processes a request with :reqmeta:`proxy` metadata, and that :reqmeta:`proxy` metadata includes proxy credentials, :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets - the ``Proxy-Authentication`` header, but only if that header is not already + the ``Proxy-Authorization`` header, but only if that header is not already set. There are third-party proxy-rotation downloader middlewares that set @@ -294,7 +294,7 @@ Scrapy 2.6.2 (2022-07-25) These third-party proxy-rotation downloader middlewares could change the :reqmeta:`proxy` metadata of a request to a new value, but fail to remove - the ``Proxy-Authentication`` header from the previous value of the + the ``Proxy-Authorization`` header from the previous value of the :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent to a different proxy. @@ -2250,7 +2250,7 @@ Scrapy 1.8.3 (2022-07-25) processes a request with :reqmeta:`proxy` metadata, and that :reqmeta:`proxy` metadata includes proxy credentials, :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets - the ``Proxy-Authentication`` header, but only if that header is not already + the ``Proxy-Authorization`` header, but only if that header is not already set. There are third-party proxy-rotation downloader middlewares that set @@ -2263,7 +2263,7 @@ Scrapy 1.8.3 (2022-07-25) These third-party proxy-rotation downloader middlewares could change the :reqmeta:`proxy` metadata of a request to a new value, but fail to remove - the ``Proxy-Authentication`` header from the previous value of the + the ``Proxy-Authorization`` header from the previous value of the :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent to a different proxy. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 40bcda288..022265992 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -652,8 +652,9 @@ per ip address instead of per domain. .. _spider-download_delay-attribute: -You can also change this setting per spider by setting ``download_delay`` -spider attribute. +.. note:: + + This delay can be set per spider using :attr:`download_delay` spider attribute. .. setting:: DOWNLOAD_HANDLERS diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index b8724a1cd..cfe25f903 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -181,8 +181,8 @@ class ImagesPipeline(FilesPipeline): stacklevel=2, ) - if image.format == "PNG" and image.mode == "RGBA": - background = self._Image.new("RGBA", image.size, (255, 255, 255)) + if image.format in ('PNG', 'WEBP') and image.mode == 'RGBA': + background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert("RGB") elif image.mode == "P": diff --git a/scrapy/shell.py b/scrapy/shell.py index 18be383c1..8e79908ab 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser +from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop class Shell: @@ -77,6 +78,10 @@ class Shell: ) def _schedule(self, request, spider): + if is_asyncio_reactor_installed(): + # set the asyncio event loop for the current thread + event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP'] + set_asyncio_event_loop(event_loop_path) spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index fea4deb4b..2560a421f 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -46,6 +46,9 @@ DEFAULT_LOGGING = { "version": 1, "disable_existing_loggers": False, "loggers": { + "filelock": { + "level": "ERROR", + }, "hpack": { "level": "ERROR", }, diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 46d83059f..e40016031 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -73,14 +73,7 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - policy = get_asyncio_event_loop_policy() - if event_loop_path is not None: - event_loop_class = load_object(event_loop_path) - event_loop = event_loop_class() - asyncio.set_event_loop(event_loop) - else: - event_loop = policy.get_event_loop() - + event_loop = set_asyncio_event_loop(event_loop_path) asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") @@ -90,6 +83,25 @@ def install_reactor(reactor_path, event_loop_path=None): installer() +def set_asyncio_event_loop(event_loop_path): + """Sets and returns the event loop with specified import path.""" + policy = get_asyncio_event_loop_policy() + if event_loop_path is not None: + event_loop_class = load_object(event_loop_path) + event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) + else: + try: + event_loop = policy.get_event_loop() + except RuntimeError: + # `get_event_loop` is expected to fail when called from a new thread + # with no asyncio event loop yet installed. Such is the case when + # called from `scrapy shell` + event_loop = policy.new_event_loop() + asyncio.set_event_loop(event_loop) + return event_loop + + def verify_installed_reactor(reactor_path): """Raises :exc:`Exception` if the installed :mod:`~twisted.internet.reactor` does not match the specified import diff --git a/setup.py b/setup.py index 27581f81d..9ce93d964 100644 --- a/setup.py +++ b/setup.py @@ -59,13 +59,14 @@ setup( "Source": "https://github.com/scrapy/scrapy", "Tracker": "https://github.com/scrapy/scrapy/issues", }, - description="A high-level Web Crawling and Web Scraping framework", - long_description=open("README.rst", encoding="utf-8").read(), - author="Scrapy developers", - maintainer="Pablo Hoffman", - maintainer_email="pablo@pablohoffman.com", - license="BSD", - packages=find_packages(exclude=("tests", "tests.*")), + description='A high-level Web Crawling and Web Scraping framework', + long_description=open('README.rst', encoding="utf-8").read(), + author='Scrapy developers', + author_email='pablo@pablohoffman.com', + maintainer='Pablo Hoffman', + maintainer_email='pablo@pablohoffman.com', + license='BSD', + packages=find_packages(exclude=('tests', 'tests.*')), include_package_data=True, zip_safe=False, entry_points={"console_scripts": ["scrapy = scrapy.cmdline:execute"]}, diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 4c4242a1b..aa5128bee 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -125,4 +125,15 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): url = "www.somedomainthatdoesntexi.st" errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False) self.assertEqual(errcode, 1, out or err) - self.assertIn(b"DNS lookup failed", err) + self.assertIn(b'DNS lookup failed', err) + + @defer.inlineCallbacks + def test_shell_fetch_async(self): + reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + url = self.url('/html') + code = f"fetch('{url}')" + args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] + _, _, err = yield self.execute(args, check_code=True) + self.assertNotIn( + b"RuntimeError: There is no current event loop in thread", err + ) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index c52fcce09..6e4971400 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,20 +2,17 @@ import codecs import unittest from unittest import mock +from packaging.version import Version as parse_version +from pytest import mark +from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding -from scrapy.http import ( - Request, - Response, - TextResponse, - HtmlResponse, - XmlResponse, - Headers, -) +from scrapy.exceptions import NotSupported +from scrapy.http import (Headers, HtmlResponse, Request, Response, + TextResponse, XmlResponse) +from scrapy.link import Link from scrapy.selector import Selector from scrapy.utils.python import to_unicode -from scrapy.exceptions import NotSupported -from scrapy.link import Link from tests import get_testdata @@ -209,13 +206,23 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com") self.assertRaises(ValueError, r.follow, None) + @mark.xfail( + parse_version(w3lib_version) < parse_version("2.1.1"), + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_whitespace_url(self): - self._assert_followed_url("foo ", "http://example.com/foo%20") + self._assert_followed_url('foo ', + 'http://example.com/foo') + @mark.xfail( + parse_version(w3lib_version) < parse_version("2.1.1"), + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_whitespace_link(self): - self._assert_followed_url( - Link("http://example.com/foo "), "http://example.com/foo%20" - ) + self._assert_followed_url(Link('http://example.com/foo '), + 'http://example.com/foo') def test_follow_flags(self): res = self.response_class("http://example.com/")