From fe60c1224e39aa3d85b20afd54566f135d9de085 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 5 Dec 2022 15:44:39 +0500 Subject: [PATCH 01/12] Fix the name of the Proxy-Authorization header in release notes (#5744) --- docs/news.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index e5fc2971a..c97de0ed8 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,7 +12,7 @@ New features ~~~~~~~~~~~~ - Relaxed the restriction introduced in 2.6.2 so that the - ``Proxy-Authentication`` header can again be set explicitly, as long as the + ``Proxy-Authorization`` header can again be set explicitly, as long as the proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and for as long as that proxy URL remains the same; this restores compatibility with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`). @@ -281,7 +281,7 @@ Scrapy 2.6.2 (2022-07-25) processes a request with :reqmeta:`proxy` metadata, and that :reqmeta:`proxy` metadata includes proxy credentials, :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets - the ``Proxy-Authentication`` header, but only if that header is not already + the ``Proxy-Authorization`` header, but only if that header is not already set. There are third-party proxy-rotation downloader middlewares that set @@ -294,7 +294,7 @@ Scrapy 2.6.2 (2022-07-25) These third-party proxy-rotation downloader middlewares could change the :reqmeta:`proxy` metadata of a request to a new value, but fail to remove - the ``Proxy-Authentication`` header from the previous value of the + the ``Proxy-Authorization`` header from the previous value of the :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent to a different proxy. @@ -2250,7 +2250,7 @@ Scrapy 1.8.3 (2022-07-25) processes a request with :reqmeta:`proxy` metadata, and that :reqmeta:`proxy` metadata includes proxy credentials, :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets - the ``Proxy-Authentication`` header, but only if that header is not already + the ``Proxy-Authorization`` header, but only if that header is not already set. There are third-party proxy-rotation downloader middlewares that set @@ -2263,7 +2263,7 @@ Scrapy 1.8.3 (2022-07-25) These third-party proxy-rotation downloader middlewares could change the :reqmeta:`proxy` metadata of a request to a new value, but fail to remove - the ``Proxy-Authentication`` header from the previous value of the + the ``Proxy-Authorization`` header from the previous value of the :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent to a different proxy. From 6757973b61766cdb28225733f1c12702fe0886db Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Sun, 11 Dec 2022 09:11:55 -0300 Subject: [PATCH 02/12] fix test_follow_whitespace_* tests --- tests/test_http_response.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index f51f3d988..cb65a6a7e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,14 +2,16 @@ import codecs import unittest from unittest import mock +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding -from scrapy.http import (Request, Response, TextResponse, HtmlResponse, - XmlResponse, Headers) +from scrapy.exceptions import NotSupported +from scrapy.http import (Headers, HtmlResponse, Request, Response, + TextResponse, XmlResponse) +from scrapy.link import Link from scrapy.selector import Selector from scrapy.utils.python import to_unicode -from scrapy.exceptions import NotSupported -from scrapy.link import Link from tests import get_testdata @@ -180,12 +182,18 @@ class BaseResponseTest(unittest.TestCase): self.assertRaises(ValueError, r.follow, None) def test_follow_whitespace_url(self): + target_url = 'http://example.com/foo' + if parse_version(w3lib_version) < parse_version("2.1.1"): + target_url += '%20' self._assert_followed_url('foo ', - 'http://example.com/foo%20') + target_url) def test_follow_whitespace_link(self): + target_url = 'http://example.com/foo' + if parse_version(w3lib_version) < parse_version("2.1.1"): + target_url += '%20' self._assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') + target_url) def test_follow_flags(self): res = self.response_class('http://example.com/') From e92e201b192609259c01b8686bb10f188e40be36 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 12 Dec 2022 07:55:15 -0300 Subject: [PATCH 03/12] Silence `filelock` logger (#5754) --- scrapy/utils/log.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 78e302d19..c8b4936ac 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -46,6 +46,9 @@ DEFAULT_LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'loggers': { + 'filelock': { + 'level': 'ERROR', + }, 'hpack': { 'level': 'ERROR', }, From 0dbd1d9b81399b624d8f8081dbc4ba08e66993e7 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 12 Dec 2022 08:58:40 -0300 Subject: [PATCH 04/12] use xfail, switch to packaging to parse version --- tests/test_http_response.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index cb65a6a7e..891b38f6e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,7 +2,8 @@ import codecs import unittest from unittest import mock -from pkg_resources import parse_version +from packaging.version import Version as parse_version +from pytest import mark from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding @@ -181,19 +182,23 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com") self.assertRaises(ValueError, r.follow, None) + @mark.xfail( + parse_version(w3lib_version) >= parse_version("2.1.1"), + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_whitespace_url(self): - target_url = 'http://example.com/foo' - if parse_version(w3lib_version) < parse_version("2.1.1"): - target_url += '%20' self._assert_followed_url('foo ', - target_url) + 'http://example.com/foo%20') + @mark.xfail( + parse_version(w3lib_version) >= parse_version("2.1.1"), + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_whitespace_link(self): - target_url = 'http://example.com/foo' - if parse_version(w3lib_version) < parse_version("2.1.1"): - target_url += '%20' self._assert_followed_url(Link('http://example.com/foo '), - target_url) + 'http://example.com/foo%20') def test_follow_flags(self): res = self.response_class('http://example.com/') From e099572cec6bb9aa7e164b3bff9fd8723286fbad Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 12 Dec 2022 09:56:50 -0300 Subject: [PATCH 05/12] change xfail condition --- tests/test_http_response.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 891b38f6e..74e170ec0 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -183,22 +183,22 @@ class BaseResponseTest(unittest.TestCase): self.assertRaises(ValueError, r.follow, None) @mark.xfail( - parse_version(w3lib_version) >= parse_version("2.1.1"), + parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, ) def test_follow_whitespace_url(self): self._assert_followed_url('foo ', - 'http://example.com/foo%20') + 'http://example.com/foo') @mark.xfail( - parse_version(w3lib_version) >= parse_version("2.1.1"), + parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, ) def test_follow_whitespace_link(self): self._assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') + 'http://example.com/foo') def test_follow_flags(self): res = self.response_class('http://example.com/') From 8b09b0e0d7eb133db648dfd02a8a531219db0a15 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 13 Dec 2022 20:41:48 -0800 Subject: [PATCH 06/12] set asyncio event loop at start of new thread --- scrapy/shell.py | 5 +++++ scrapy/utils/reactor.py | 27 +++++++++++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/scrapy/shell.py b/scrapy/shell.py index f2dff2ae3..515b71bb6 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser +from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop class Shell: @@ -76,6 +77,10 @@ class Shell: banner=self.vars.pop('banner', '')) def _schedule(self, request, spider): + if is_asyncio_reactor_installed(): + # set the asyncio event loop for the current thread + event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP'] + set_asyncio_event_loop(event_loop_path) spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ddf354d88..532d7b0d0 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -71,14 +71,7 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - policy = get_asyncio_event_loop_policy() - if event_loop_path is not None: - event_loop_class = load_object(event_loop_path) - event_loop = event_loop_class() - asyncio.set_event_loop(event_loop) - else: - event_loop = policy.get_event_loop() - + event_loop = set_asyncio_event_loop(event_loop_path) asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") @@ -87,6 +80,24 @@ def install_reactor(reactor_path, event_loop_path=None): with suppress(error.ReactorAlreadyInstalledError): installer() +def set_asyncio_event_loop(event_loop_path): + """Sets and returns the event loop with specified import path.""" + policy = get_asyncio_event_loop_policy() + if event_loop_path is not None: + event_loop_class = load_object(event_loop_path) + event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) + else: + try: + event_loop = policy.get_event_loop() + except RuntimeError: + # `get_event_loop` is expected to fail when called from a new thread + # with no asyncio event loop yet installed. Such is the case when + # called from `scrapy shell` + event_loop = policy.new_event_loop() + asyncio.set_event_loop(event_loop) + return event_loop + def verify_installed_reactor(reactor_path): """Raises :exc:`Exception` if the installed From c2de9372a27120abf81f2aaf6609e5acae82549e Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 13 Dec 2022 21:37:17 -0800 Subject: [PATCH 07/12] added unittest for scrapy shell command and fixed flake8 error --- scrapy/utils/reactor.py | 1 + tests/test_command_shell.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 532d7b0d0..e6b8de292 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -80,6 +80,7 @@ def install_reactor(reactor_path, event_loop_path=None): with suppress(error.ReactorAlreadyInstalledError): installer() + def set_asyncio_event_loop(event_loop_path): """Sets and returns the event loop with specified import path.""" policy = get_asyncio_event_loop_policy() diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 33c98ad69..f06a02f5f 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -115,3 +115,14 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'DNS lookup failed', err) + + @defer.inlineCallbacks + def test_shell_fetch_async(self): + reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + url = self.url('/html') + code = f"fetch('{url}')" + args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] + _, _, err = yield self.execute(args, check_code=True) + self.assertNotIn( + b"RuntimeError: There is no current event loop in thread", err + ) From 96e526aad8ac93966e383a054d151e6561270bc7 Mon Sep 17 00:00:00 2001 From: Yohanna Lisnichuk Date: Thu, 15 Dec 2022 10:01:55 -0300 Subject: [PATCH 08/12] docs: change DOWNLOAD_DELAY description format To be consistent with the other settings' descriptions --- docs/topics/settings.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 40bcda288..c935e74a7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -652,8 +652,10 @@ per ip address instead of per domain. .. _spider-download_delay-attribute: -You can also change this setting per spider by setting ``download_delay`` -spider attribute. +Note + +This delay can be set per spider using :attr:`download_delay` spider attribute. + .. setting:: DOWNLOAD_HANDLERS From b63ca6f834d06ba9b63108079129705f6326354c Mon Sep 17 00:00:00 2001 From: Yohanna Lisnichuk Date: Thu, 15 Dec 2022 13:20:45 -0300 Subject: [PATCH 09/12] docs: use the right notes directive --- docs/topics/settings.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c935e74a7..022265992 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -652,10 +652,9 @@ per ip address instead of per domain. .. _spider-download_delay-attribute: -Note - -This delay can be set per spider using :attr:`download_delay` spider attribute. +.. note:: + This delay can be set per spider using :attr:`download_delay` spider attribute. .. setting:: DOWNLOAD_HANDLERS From 2e54237649bc8301a1c44ab60c6d7908eb9f71cc Mon Sep 17 00:00:00 2001 From: Kyle Reynolds Date: Tue, 20 Dec 2022 17:54:06 -0700 Subject: [PATCH 10/12] Update setup.py adding author_email to match_maintainer email for thoroughness' sake --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 82ac86cdd..a00e86bab 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ setup( description='A high-level Web Crawling and Web Scraping framework', long_description=open('README.rst', encoding="utf-8").read(), author='Scrapy developers', + author_email='Pablo Hoffman', maintainer='Pablo Hoffman', maintainer_email='pablo@pablohoffman.com', license='BSD', From 3b5ce4c182d6b243645b9ce27d5f5084dbc4cc88 Mon Sep 17 00:00:00 2001 From: Kyle Reynolds Date: Wed, 21 Dec 2022 10:51:50 -0700 Subject: [PATCH 11/12] Update setup.py Sorry about the goof up. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a00e86bab..27445cad0 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ setup( description='A high-level Web Crawling and Web Scraping framework', long_description=open('README.rst', encoding="utf-8").read(), author='Scrapy developers', - author_email='Pablo Hoffman', + author_email='pablo@pablohoffman.com', maintainer='Pablo Hoffman', maintainer_email='pablo@pablohoffman.com', license='BSD', From 7ae32ea38d9b78402528ac3dffc8e1c5f1cf86b7 Mon Sep 17 00:00:00 2001 From: Steve Niafas Date: Thu, 22 Dec 2022 15:33:14 +0200 Subject: [PATCH 12/12] Fix transparency for processed webp images (#5767) --- scrapy/pipelines/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 6a28a3b87..d92f24d56 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -171,7 +171,7 @@ class ImagesPipeline(FilesPipeline): 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=2) - if image.format == 'PNG' and image.mode == 'RGBA': + if image.format in ('PNG', 'WEBP') and image.mode == 'RGBA': background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB')