From 19022849428573a9bdf5f3217638d6e3c86d1796 Mon Sep 17 00:00:00 2001 From: Chan Sau Yee <15137352+y26805@users.noreply.github.com> Date: Fri, 29 Dec 2023 20:32:51 +0900 Subject: [PATCH 01/21] Update black reference in docs (#6192) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 2b3249601..d728338da 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -178,7 +178,7 @@ Scrapy: * We use `black `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e black``. + run black manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. From 40e623b2768598e36c4f367bd166b36fffceb3f6 Mon Sep 17 00:00:00 2001 From: Chan Sau Yee <15137352+y26805@users.noreply.github.com> Date: Fri, 29 Dec 2023 20:33:37 +0900 Subject: [PATCH 02/21] Add type hints (#6191) --- scrapy/pipelines/files.py | 4 +++- scrapy/pipelines/images.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 5c09ab37e..1990ba825 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -340,7 +340,9 @@ class FilesPipeline(MediaPipeline): DEFAULT_FILES_URLS_FIELD = "file_urls" DEFAULT_FILES_RESULT_FIELD = "files" - def __init__(self, store_uri, download_func=None, settings=None): + def __init__( + self, store_uri: Union[str, PathLike], download_func=None, settings=None + ): store_uri = _to_string(store_uri) if not store_uri: raise NotConfigured diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 1bd9832a8..02c4b1361 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -8,7 +8,8 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import Dict, Tuple +from os import PathLike +from typing import Dict, Tuple, Union from itemadapter import ItemAdapter @@ -53,7 +54,9 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" - def __init__(self, store_uri, download_func=None, settings=None): + def __init__( + self, store_uri: Union[str, PathLike], download_func=None, settings=None + ): try: from PIL import Image From c7b2b097b18c0b30d06cea4b803d5d42aca98715 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Fri, 5 Jan 2024 10:50:45 +0100 Subject: [PATCH 03/21] fix(typo): correct `successfully` --- tests/test_extension_periodic_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 502ada6be..b7312bbcd 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -67,7 +67,7 @@ def extension(settings=None): class TestPeriodicLog(unittest.TestCase): def test_extension_enabled(self): - # Expected that settings for this extension loaded succesfully + # Expected that settings for this extension loaded successfully # And on certain conditions - extension raising NotConfigured # "PERIODIC_LOG_STATS": True -> set to {"enabled": True} From 0d445a3224ecb0abfdf56a93e181692d3b5e4a6b Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Fri, 5 Jan 2024 12:30:10 +0100 Subject: [PATCH 04/21] refactor(yield): use `yield from` syntax --- scrapy/core/spidermw.py | 3 +-- scrapy/spiders/crawl.py | 3 +-- scrapy/spiders/feed.py | 6 ++---- scrapy/spiders/sitemap.py | 3 +-- scrapy/utils/python.py | 3 +-- scrapy/utils/request.py | 3 +-- tests/spiders.py | 3 +-- tests/test_feedexport.py | 6 ++---- tests/test_spider.py | 3 +-- tests/test_spidermiddleware.py | 9 +++------ tests/test_utils_defer.py | 3 +-- 11 files changed, 15 insertions(+), 30 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index dcf1a6dbc..031a0be36 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -103,8 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Union[Generator, AsyncGenerator]: def process_sync(iterable: Iterable) -> Generator: try: - for r in iterable: - yield r + yield from iterable except Exception as ex: exception_result = self._process_spider_exception( response, spider, Failure(ex), exception_processor_index diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 31e845716..ebb4f5984 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -131,8 +131,7 @@ class CrawlSpider(Spider): def _handle_failure(self, failure, errback): if errback: results = errback(failure) or () - for request_or_item in iterate_spider_output(results): - yield request_or_item + yield from iterate_spider_output(results) def _compile_rules(self): self._rules = [] diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6afadc577..47827e442 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -58,8 +58,7 @@ class XMLFeedSpider(Spider): for selector in nodes: ret = iterate_spider_output(self.parse_node(response, selector)) - for result_item in self.process_results(response, ret): - yield result_item + yield from self.process_results(response, ret) def _parse(self, response, **kwargs): if not hasattr(self, "parse_node"): @@ -133,8 +132,7 @@ class CSVFeedSpider(Spider): response, self.delimiter, self.headers, quotechar=self.quotechar ): ret = iterate_spider_output(self.parse_row(response, row)) - for result_item in self.process_results(response, ret): - yield result_item + yield from self.process_results(response, ret) def _parse(self, response, **kwargs): if not hasattr(self, "parse_row"): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index aaf75a519..974665fe0 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -33,8 +33,7 @@ class SitemapSpider(Spider): attributes, for example, you can filter locs with lastmod greater than a given date (see docs). """ - for entry in entries: - yield entry + yield from entries def _parse_sitemap(self, response): if response.url.endswith("/robots.txt"): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 0b5dc324f..68ca96b69 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -57,8 +57,7 @@ def iflatten(x: Iterable) -> Iterable: Similar to ``.flatten()``, but returns iterator instead""" for el in x: if is_listlike(el): - for el_ in iflatten(el): - yield el_ + yield from iflatten(el) else: yield el diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 24fcbd85e..cea1bc727 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -44,8 +44,7 @@ def _serialize_headers( for header in headers: if header in request.headers: yield header - for value in request.headers.getlist(header): - yield value + yield from request.headers.getlist(header) def request_fingerprint( diff --git a/tests/spiders.py b/tests/spiders.py index f29dea2a1..3df153a12 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -301,8 +301,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): def parse(self, response): self.seedsseen.append(response.meta.get("seed")) - for req in super().parse(response): - yield req + yield from super().parse(response) class SingleRequestSpider(MetaSpider): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 89169fd7c..c7d955bc7 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -673,8 +673,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): name = "testspider" def parse(self, response): - for item in items: - yield item + yield from items data = yield self.run_and_export(TestSpider, settings) return data @@ -2696,8 +2695,7 @@ class BatchDeliveriesTest(FeedExportTestBase): name = "testspider" def parse(self, response): - for item in items: - yield item + yield from items with MockServer() as server: TestSpider.start_urls = [server.url("/")] diff --git a/tests/test_spider.py b/tests/test_spider.py index 00da3d485..9ce40f921 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -317,8 +317,7 @@ class CrawlSpiderTest(SpiderTest): rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),) def dummy_process_links(self, links): - for link in links: - yield link + yield from links spider = _CrawlSpider() output = list(spider._requests_to_follow(response)) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index d167adbb7..38ca8d950 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -170,8 +170,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): class ProcessSpiderOutputSimpleMiddleware: def process_spider_output(self, response, result, spider): - for r in result: - yield r + yield from result class ProcessSpiderOutputAsyncGenMiddleware: @@ -182,8 +181,7 @@ class ProcessSpiderOutputAsyncGenMiddleware: class ProcessSpiderOutputUniversalMiddleware: def process_spider_output(self, response, result, spider): - for r in result: - yield r + yield from result async def process_spider_output_async(self, response, result, spider): async for r in result: @@ -324,8 +322,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): class ProcessStartRequestsSimpleMiddleware: def process_start_requests(self, start_requests, spider): - for r in start_requests: - yield r + yield from start_requests class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index bb0ebc2a4..a7d54b565 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -107,8 +107,7 @@ class DeferUtilsTest(unittest.TestCase): class IterErrbackTest(unittest.TestCase): def test_iter_errback_good(self): def itergood(): - for x in range(10): - yield x + yield from range(10) errors = [] out = list(iter_errback(itergood(), errors.append)) From 42c481cb4a12a81e88923930e21a661eee967a5f Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Fri, 5 Jan 2024 12:36:36 +0100 Subject: [PATCH 05/21] refactor(): use `OSError` exception https://docs.astral.sh/ruff/rules/os-error-alias/ --- scrapy/pipelines/files.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1990ba825..73064ad10 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -8,7 +8,6 @@ import functools import hashlib import logging import mimetypes -import os import time from collections import defaultdict from contextlib import suppress @@ -66,7 +65,7 @@ class FSFilesStore: absolute_path = self._get_filesystem_path(path) try: last_modified = absolute_path.stat().st_mtime - except os.error: + except OSError: return {} with absolute_path.open("rb") as f: From 745b8412f6ec02605c27d295b2be9d71b624cf70 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Fri, 5 Jan 2024 14:53:51 +0100 Subject: [PATCH 06/21] fix(flake8): lint errors E226 missing whitespace around arithmetic operator E201 whitespace after '{' --- scrapy/extensions/memusage.py | 6 +++--- tests/test_utils_iterators.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 221967bda..ca766c938 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -128,9 +128,9 @@ class MemoryUsage: def _send_report(self, rcpts, subject): """send notification mail with some additional useful info""" stats = self.crawler.stats - s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n" - s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n" - s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n" + s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" + s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n" + s += f"Current memory usage : {self.get_virtual_size() / 1024 / 1024}M\r\n" s += ( "ENGINE STATUS ------------------------------------------------------- \r\n" diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 3598fa0bb..4a0c34d82 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -524,6 +524,6 @@ class TestHelper(unittest.TestCase): def _assert_type_and_value(self, a, b, obj): self.assertTrue( - type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}" + type(a) is type(b), f"Got {type(a)}, expected {type(b)} for {obj!r}" ) self.assertEqual(a, b) From 68fccb1d58f291f70e864fd8ecd167887bda4112 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 6 Jan 2024 01:35:56 +0400 Subject: [PATCH 07/21] Fix and re-enable newer mitmproxy usage in tests. --- tests/test_proxy_connect.py | 3 ++- tox.ini | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index dc0a82086..46d42e9f6 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -31,6 +31,7 @@ sys.exit(mitmdump()) self.proc = Popen( [ sys.executable, + "-u", "-c", script, "--listen-host", @@ -46,7 +47,7 @@ sys.exit(mitmdump()) stdout=PIPE, ) line = self.proc.stdout.readline().decode("utf-8") - host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1) + host_port = re.search(r"listening at (?:http://)?([^:]+:\d+)", line).group(1) address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}" return address diff --git a/tox.ini b/tox.ini index 932c0b805..d9dcacc01 100644 --- a/tox.ini +++ b/tox.ini @@ -11,11 +11,7 @@ minversion = 1.7.0 deps = -rtests/requirements.txt # mitmproxy does not support PyPy - # Python 3.9+ requires mitmproxy >= 5.3.0 - # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 - #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' - # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 - mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' + mitmproxy; implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -87,7 +83,7 @@ deps = lxml==4.4.1 -rtests/requirements.txt - # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies + # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment setenv = _SCRAPY_PINNED=true From c2baf4d0dad5f656e51dce6e00118fbc0419d0db Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 12 Jan 2024 18:30:41 +0400 Subject: [PATCH 08/21] Remove a defer.returnValue call. --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c7d955bc7..277555608 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2299,7 +2299,7 @@ class BatchDeliveriesTest(FeedExportTestBase): content[feed["format"]].append(file.read_bytes()) finally: self.tearDown() - defer.returnValue(content) + return content @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): From fa0c598096de6e26a7b22e7d53cf8c073f96f3a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 15 Jan 2024 13:14:02 +0100 Subject: [PATCH 09/21] Add component getters to Crawler (#6181) --- scrapy/crawler.py | 42 +++++ tests/test_crawler.py | 388 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 419 insertions(+), 11 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 844d5f759..1db9ace28 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -178,6 +178,48 @@ class Crawler: assert self.engine yield maybeDeferred(self.engine.stop) + @staticmethod + def _get_component(component_class, components): + for component in components: + if isinstance(component, component_class): + return component + return None + + def get_addon(self, cls): + return self._get_component(cls, self.addons.addons) + + def get_downloader_middleware(self, cls): + if not self.engine: + raise RuntimeError( + "Crawler.get_downloader_middleware() can only be called after " + "the crawl engine has been created." + ) + return self._get_component(cls, self.engine.downloader.middleware.middlewares) + + def get_extension(self, cls): + if not self.extensions: + raise RuntimeError( + "Crawler.get_extension() can only be called after the " + "extension manager has been created." + ) + return self._get_component(cls, self.extensions.middlewares) + + def get_item_pipeline(self, cls): + if not self.engine: + raise RuntimeError( + "Crawler.get_item_pipeline() can only be called after the " + "crawl engine has been created." + ) + return self._get_component(cls, self.engine.scraper.itemproc.middlewares) + + def get_spider_middleware(self, cls): + if not self.engine: + raise RuntimeError( + "Crawler.get_spider_middleware() can only be called after the " + "crawl engine has been created." + ) + return self._get_component(cls, self.engine.scraper.spidermw.middlewares) + class CrawlerRunner: """ diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 92bd5f38f..989208694 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -12,12 +12,13 @@ import pytest from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from pytest import mark, raises -from twisted.internet import defer +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.trial import unittest from w3lib import __version__ as w3lib_version from zope.interface.exceptions import MultipleInvalid import scrapy +from scrapy import Spider from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions import telnet @@ -29,6 +30,19 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer, get_mockserver_env +# To prevent warnings. +BASE_SETTINGS = { + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", +} + + +def get_raw_crawler(spidercls=None, settings_dict=None): + """get_crawler alternative that only calls the __init__ method of the + crawler.""" + settings = Settings() + settings.setdict(settings_dict or {}) + return Crawler(spidercls or DefaultSpider, settings) + class BaseCrawlerTest(unittest.TestCase): def assertOptionIsDefault(self, settings, key): @@ -39,7 +53,7 @@ class BaseCrawlerTest(unittest.TestCase): class CrawlerTestCase(BaseCrawlerTest): def test_populate_spidercls_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} + project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"} class CustomSettingsSpider(DefaultSpider): custom_settings = spider_settings @@ -71,9 +85,9 @@ class CrawlerTestCase(BaseCrawlerTest): with raises(ValueError): Crawler(DefaultSpider()) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_crawl_twice_deprecated(self): - crawler = Crawler(NoRequestsSpider) + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() with pytest.warns( ScrapyDeprecationWarning, @@ -81,6 +95,358 @@ class CrawlerTestCase(BaseCrawlerTest): ): yield crawler.crawl() + def test_get_addon(self): + class ParentAddon: + pass + + class TrackingAddon(ParentAddon): + instances = [] + + def __init__(self): + TrackingAddon.instances.append(self) + + def update_settings(self, settings): + pass + + settings = { + **BASE_SETTINGS, + "ADDONS": { + TrackingAddon: 0, + }, + } + crawler = get_crawler(settings_dict=settings) + self.assertEqual(len(TrackingAddon.instances), 1) + expected = TrackingAddon.instances[-1] + + addon = crawler.get_addon(TrackingAddon) + self.assertEqual(addon, expected) + + addon = crawler.get_addon(DefaultSpider) + self.assertIsNone(addon) + + addon = crawler.get_addon(ParentAddon) + self.assertEqual(addon, expected) + + class ChildAddon(TrackingAddon): + pass + + addon = crawler.get_addon(ChildAddon) + self.assertIsNone(addon) + + @inlineCallbacks + def test_get_downloader_middleware(self): + class ParentDownloaderMiddleware: + pass + + class TrackingDownloaderMiddleware(ParentDownloaderMiddleware): + instances = [] + + def __init__(self): + TrackingDownloaderMiddleware.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_downloader_middleware(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "DOWNLOADER_MIDDLEWARES": { + TrackingDownloaderMiddleware: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingDownloaderMiddleware + yield crawler.crawl() + self.assertEqual(len(TrackingDownloaderMiddleware.instances), 1) + self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentDownloaderMiddleware + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingDownloaderMiddleware.instances[-1]) + + class ChildDownloaderMiddleware(TrackingDownloaderMiddleware): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildDownloaderMiddleware + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_downloader_middleware_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises( + RuntimeError, crawler.get_downloader_middleware, DefaultSpider + ) + + @inlineCallbacks + def test_get_downloader_middleware_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_downloader_middleware(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + + @inlineCallbacks + def test_get_extension(self): + class ParentExtension: + pass + + class TrackingExtension(ParentExtension): + instances = [] + + def __init__(self): + TrackingExtension.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_extension(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "EXTENSIONS": { + TrackingExtension: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingExtension + yield crawler.crawl() + self.assertEqual(len(TrackingExtension.instances), 1) + self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentExtension + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingExtension.instances[-1]) + + class ChildExtension(TrackingExtension): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildExtension + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_extension_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises(RuntimeError, crawler.get_extension, DefaultSpider) + + @inlineCallbacks + def test_get_extension_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_extension(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + + @inlineCallbacks + def test_get_item_pipeline(self): + class ParentItemPipeline: + pass + + class TrackingItemPipeline(ParentItemPipeline): + instances = [] + + def __init__(self): + TrackingItemPipeline.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_item_pipeline(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "ITEM_PIPELINES": { + TrackingItemPipeline: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingItemPipeline + yield crawler.crawl() + self.assertEqual(len(TrackingItemPipeline.instances), 1) + self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentItemPipeline + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingItemPipeline.instances[-1]) + + class ChildItemPipeline(TrackingItemPipeline): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildItemPipeline + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_item_pipeline_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises(RuntimeError, crawler.get_item_pipeline, DefaultSpider) + + @inlineCallbacks + def test_get_item_pipeline_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_item_pipeline(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + + @inlineCallbacks + def test_get_spider_middleware(self): + class ParentSpiderMiddleware: + pass + + class TrackingSpiderMiddleware(ParentSpiderMiddleware): + instances = [] + + def __init__(self): + TrackingSpiderMiddleware.instances.append(self) + + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) + + def __init__(self, crawler): + self.crawler = crawler + + def start_requests(self): + MySpider.result = crawler.get_spider_middleware(MySpider.cls) + return + yield + + settings = { + **BASE_SETTINGS, + "SPIDER_MIDDLEWARES": { + TrackingSpiderMiddleware: 0, + }, + } + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = TrackingSpiderMiddleware + yield crawler.crawl() + self.assertEqual(len(TrackingSpiderMiddleware.instances), 1) + self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = DefaultSpider + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ParentSpiderMiddleware + yield crawler.crawl() + self.assertEqual(MySpider.result, TrackingSpiderMiddleware.instances[-1]) + + class ChildSpiderMiddleware(TrackingSpiderMiddleware): + pass + + crawler = get_raw_crawler(MySpider, settings) + MySpider.cls = ChildSpiderMiddleware + yield crawler.crawl() + self.assertIsNone(MySpider.result) + + def test_get_spider_middleware_not_crawling(self): + crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) + self.assertRaises(RuntimeError, crawler.get_spider_middleware, DefaultSpider) + + @inlineCallbacks + def test_get_spider_middleware_no_engine(self): + class MySpider(Spider): + name = "myspider" + + @classmethod + def from_crawler(cls, crawler): + try: + crawler.get_spider_middleware(DefaultSpider) + except Exception as e: + MySpider.result = e + raise + + crawler = get_raw_crawler(MySpider, BASE_SETTINGS) + with raises(RuntimeError): + yield crawler.crawl() + class SpiderSettingsTestCase(unittest.TestCase): def test_spider_custom_settings(self): @@ -223,20 +589,20 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"}) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) self.assertFalse(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) self.assertFalse(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_failed(self): runner = self._runner() @@ -249,7 +615,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertTrue(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): runner = self._runner() @@ -264,7 +630,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertTrue(runner.bootstrap_failed) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == "asyncio": CrawlerRunner( @@ -531,7 +897,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.expect_exact("Spider closed (shutdown)") p.wait() - @defer.inlineCallbacks + @inlineCallbacks def test_shutdown_forced(self): from twisted.internet import reactor @@ -543,7 +909,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.kill(sig) p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems - d = defer.Deferred() + d = Deferred() reactor.callLater(0.1, d.callback, None) yield d p.kill(sig) From e8dadb959219afea1d3a3f67ce03ac3c7a51520c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 15 Jan 2024 13:37:03 +0100 Subject: [PATCH 10/21] scrapy parse: fix the signature of callbacks from the CLI (#6182) --- scrapy/commands/parse.py | 59 ++++++++++++++++++++----------------- tests/test_command_check.py | 4 +-- tests/test_command_parse.py | 18 ++++++++++- 3 files changed, 51 insertions(+), 30 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ac937e464..c9f8586d3 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,3 +1,4 @@ +import functools import inspect import json import logging @@ -251,39 +252,40 @@ class Command(BaseRunSpiderCommand): return scraped_data + def _get_callback(self, *, spider, opts, response=None): + cb = None + if response: + cb = response.meta["_callback"] + if not cb: + if opts.callback: + cb = opts.callback + elif response and opts.rules and self.first_response == response: + cb = self.get_callback_from_rules(spider, response) + if not cb: + raise ValueError( + f"Cannot find a rule that matches {response.url!r} in spider: " + f"{spider.name}" + ) + else: + cb = "parse" + + if not callable(cb): + cb_method = getattr(spider, cb, None) + if callable(cb_method): + cb = cb_method + else: + raise ValueError( + f"Cannot find callback {cb!r} in spider: {spider.name}" + ) + return cb + def prepare_request(self, spider, request, opts): def callback(response, **cb_kwargs): # memorize first request if not self.first_response: self.first_response = response - # determine real callback - cb = response.meta["_callback"] - if not cb: - if opts.callback: - cb = opts.callback - elif opts.rules and self.first_response == response: - cb = self.get_callback_from_rules(spider, response) - - if not cb: - logger.error( - "Cannot find a rule that matches %(url)r in spider: %(spider)s", - {"url": response.url, "spider": spider.name}, - ) - return - else: - cb = "parse" - - if not callable(cb): - cb_method = getattr(spider, cb, None) - if callable(cb_method): - cb = cb_method - else: - logger.error( - "Cannot find callback %(callback)r in spider: %(spider)s", - {"callback": cb, "spider": spider.name}, - ) - return + cb = self._get_callback(spider=spider, opts=opts, response=response) # parse items and requests depth = response.meta["_depth"] @@ -303,6 +305,9 @@ class Command(BaseRunSpiderCommand): request.meta["_depth"] = 1 request.meta["_callback"] = request.callback + if not request.callback and not opts.rules: + cb = self._get_callback(spider=spider, opts=opts) + functools.update_wrapper(callback, cb) request.callback = callback return request diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 129ef0121..592494aba 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -16,11 +16,11 @@ import scrapy class CheckSpider(scrapy.Spider): name = '{self.spider_name}' - start_urls = ['http://toscrape.com'] + start_urls = ['data:,'] def parse(self, response, **cb_kwargs): \"\"\" - @url http://toscrape.com + @url data:, {contracts} \"\"\" {parse_def} diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 037333c03..9356d6b79 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -78,9 +78,21 @@ class AsyncDefAsyncioGenExcSpider(scrapy.Spider): if i > 5: raise ValueError("Stopping the processing") +class CallbackSignatureDownloaderMiddleware: + def process_request(self, request, spider): + from inspect import signature + spider.logger.debug(f"request.callback signature: {{signature(request.callback)}}") + + class MySpider(scrapy.Spider): name = '{self.spider_name}' + custom_settings = {{ + "DOWNLOADER_MIDDLEWARES": {{ + CallbackSignatureDownloaderMiddleware: 0, + }} + }} + def parse(self, response): if getattr(self, 'test_arg', None): self.logger.debug('It Works!') @@ -220,7 +232,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url("/html"), ] ) - self.assertIn("DEBUG: It Works!", _textmode(stderr)) + log = _textmode(stderr) + self.assertIn("DEBUG: It Works!", log) + self.assertIn( + "DEBUG: request.callback signature: (response, foo=None, key=None)", log + ) @defer.inlineCallbacks def test_request_without_meta(self): From d5233bb57f35c54b0c03981dc59c7328ca44cb9a Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Mon, 15 Jan 2024 14:11:33 +0100 Subject: [PATCH 11/21] chore(docs): update `sphinx` dependencies (#6200) --- docs/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 9f9aef711..5f683d34c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -sphinx==5.0.2 -sphinx-hoverxref==1.1.1 -sphinx-notfound-page==0.8 -sphinx-rtd-theme==1.0.0 +sphinx==6.2.1 +sphinx-hoverxref==1.3.0 +sphinx-notfound-page==1.0.0 +sphinx-rtd-theme==2.0.0 From 88285e75b6b7a22689208c2d321a1eda60b64003 Mon Sep 17 00:00:00 2001 From: Kevin Toms Date: Wed, 17 Jan 2024 10:05:22 -0500 Subject: [PATCH 12/21] Add FAQ on making a blank request --- docs/faq.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 20dd814df..9df4490d4 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -405,6 +405,25 @@ or :class:`~scrapy.signals.headers_received` signals and raising a :ref:`topics-stop-response-download` topic for additional information and examples. +.. _faq-blank-request: + +How can I make a blank request? +------------------------------- + +.. code-block:: python + + from scrapy import Request + + yield Request( + url="data:,", + callback=self.your_call_back, + ) + +In this case, the URL is set to a data URI scheme. Data URLs allow you to include data +in-line in web pages as if they were external resources. The "data:" scheme with an empty +content (",") essentially creates a request to a data URL without any specific content. + + Running ``runspider`` I get ``error: No spider found in file: `` -------------------------------------------------------------------------- From 46f94ec9cb0f480999f018ceab4a5751abaf180e Mon Sep 17 00:00:00 2001 From: Kevin Toms Date: Wed, 17 Jan 2024 15:49:51 -0500 Subject: [PATCH 13/21] Fix test Wrap the yield line in a function to prevent throwing error when the code snippet is executed --- docs/faq.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 9df4490d4..0282fc6e2 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -414,10 +414,11 @@ How can I make a blank request? from scrapy import Request - yield Request( - url="data:,", - callback=self.your_call_back, - ) + def make_blank_request(your_call_back): + yield Request( + url="data:,", + callback=your_call_back, + ) In this case, the URL is set to a data URI scheme. Data URLs allow you to include data in-line in web pages as if they were external resources. The "data:" scheme with an empty From 9074c16497bde04f5d561df1d584abe2cc73f183 Mon Sep 17 00:00:00 2001 From: Kevin Toms Date: Thu, 18 Jan 2024 09:36:25 -0500 Subject: [PATCH 14/21] make suggestion --- docs/faq.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 0282fc6e2..2113b0964 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -414,11 +414,8 @@ How can I make a blank request? from scrapy import Request - def make_blank_request(your_call_back): - yield Request( - url="data:,", - callback=your_call_back, - ) + + blank_request = Request("data:,") In this case, the URL is set to a data URI scheme. Data URLs allow you to include data in-line in web pages as if they were external resources. The "data:" scheme with an empty From 2487e3cc035e490777b921badc84c11b9fb77b20 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 10:05:50 -0300 Subject: [PATCH 15/21] Cleanup deprecated fingerprint code in scrapy.utils.request --- scrapy/settings/default_settings.py | 2 +- scrapy/utils/request.py | 143 +-------------- tests/test_utils_request.py | 262 +--------------------------- 3 files changed, 6 insertions(+), 401 deletions(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d6b3585e2..02494bad0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -260,7 +260,7 @@ REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.6" +REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index cea1bc727..b230cd214 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -5,7 +5,6 @@ scrapy.http.Request objects import hashlib import json -import warnings from typing import ( TYPE_CHECKING, Any, @@ -26,7 +25,6 @@ from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url from scrapy import Request, Spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -34,9 +32,6 @@ from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: from scrapy.crawler import Crawler -_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" -_deprecated_fingerprint_cache = WeakKeyDictionary() - def _serialize_headers( headers: Iterable[bytes], request: Request @@ -47,120 +42,6 @@ def _serialize_headers( yield from request.headers.getlist(header) -def request_fingerprint( - request: Request, - include_headers: Optional[Iterable[Union[bytes, str]]] = None, - keep_fragments: bool = False, -) -> str: - """ - Return the request fingerprint as an hexadecimal string. - - The request fingerprint is a hash that uniquely identifies the resource the - request points to. For example, take the following two urls: - - http://www.example.com/query?id=111&cat=222 - http://www.example.com/query?cat=222&id=111 - - Even though those are two different URLs both point to the same resource - and are equivalent (i.e. they should return the same response). - - Another example are cookies used to store session ids. Suppose the - following page is only accessible to authenticated users: - - http://www.example.com/members/offers.html - - Lots of sites use a cookie to store the session id, which adds a random - component to the HTTP Request and thus should be ignored when calculating - the fingerprint. - - For this reason, request headers are ignored by default when calculating - the fingerprint. If you want to include specific headers use the - include_headers argument, which is a list of Request headers to include. - - Also, servers usually ignore fragments in urls when handling requests, - so they are also ignored by default when calculating the fingerprint. - If you want to include them, set the keep_fragments argument to True - (for instance when handling requests with a headless browser). - """ - if include_headers or keep_fragments: - message = ( - "Call to deprecated function " - "scrapy.utils.request.request_fingerprint().\n" - "\n" - "If you are using this function in a Scrapy component because you " - "need a non-default fingerprinting algorithm, and you are OK " - "with that non-default fingerprinting algorithm being used by " - "all Scrapy components and not just the one calling this " - "function, use crawler.request_fingerprinter.fingerprint() " - "instead in your Scrapy component (you can get the crawler " - "object from the 'from_crawler' class method), and use the " - "'REQUEST_FINGERPRINTER_CLASS' setting to configure your " - "non-default fingerprinting algorithm.\n" - "\n" - "Otherwise, consider using the " - "scrapy.utils.request.fingerprint() function instead.\n" - "\n" - "If you switch to 'fingerprint()', or assign the " - "'REQUEST_FINGERPRINTER_CLASS' setting a class that uses " - "'fingerprint()', the generated fingerprints will not only be " - "bytes instead of a string, but they will also be different from " - "those generated by 'request_fingerprint()'. Before you switch, " - "make sure that you understand the consequences of this (e.g. " - "cache invalidation) and are OK with them; otherwise, consider " - "implementing your own function which returns the same " - "fingerprints as the deprecated 'request_fingerprint()' function." - ) - else: - message = ( - "Call to deprecated function " - "scrapy.utils.request.request_fingerprint().\n" - "\n" - "If you are using this function in a Scrapy component, and you " - "are OK with users of your component changing the fingerprinting " - "algorithm through settings, use " - "crawler.request_fingerprinter.fingerprint() instead in your " - "Scrapy component (you can get the crawler object from the " - "'from_crawler' class method).\n" - "\n" - "Otherwise, consider using the " - "scrapy.utils.request.fingerprint() function instead.\n" - "\n" - "Either way, the resulting fingerprints will be returned as " - "bytes, not as a string, and they will also be different from " - "those generated by 'request_fingerprint()'. Before you switch, " - "make sure that you understand the consequences of this (e.g. " - "cache invalidation) and are OK with them; otherwise, consider " - "implementing your own function which returns the same " - "fingerprints as the deprecated 'request_fingerprint()' function." - ) - warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) - processed_include_headers: Optional[Tuple[bytes, ...]] = None - if include_headers: - processed_include_headers = tuple( - to_bytes(h.lower()) for h in sorted(include_headers) - ) - cache = _deprecated_fingerprint_cache.setdefault(request, {}) - cache_key = (processed_include_headers, keep_fragments) - if cache_key not in cache: - fp = hashlib.sha1() - fp.update(to_bytes(request.method)) - fp.update( - to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) - ) - fp.update(request.body or b"") - if processed_include_headers: - for part in _serialize_headers(processed_include_headers, request): - fp.update(part) - cache[cache_key] = fp.hexdigest() - return cache[cache_key] - - -def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - return bytes.fromhex(request_fingerprint(*args, **kwargs)) - - _fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" _fingerprint_cache = WeakKeyDictionary() @@ -258,32 +139,14 @@ class RequestFingerprinter: "REQUEST_FINGERPRINTER_IMPLEMENTATION" ) else: - implementation = "2.6" - if implementation == "2.6": - message = ( - "'2.6' is a deprecated value for the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting.\n" - "\n" - "It is also the default value. In other words, it is normal " - "to get this warning if you have not defined a value for the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting. This is so " - "for backward compatibility reasons, but it will change in a " - "future version of Scrapy.\n" - "\n" - "See the documentation of the " - "'REQUEST_FINGERPRINTER_IMPLEMENTATION' setting for " - "information on how to handle this deprecation." - ) - warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) - self._fingerprint = _request_fingerprint_as_bytes - elif implementation == "2.7": + implementation = "2.7" + if implementation == "2.7": self._fingerprint = fingerprint else: raise ValueError( f"Got an invalid value on setting " f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': " - f"{implementation!r}. Valid values are '2.6' (deprecated) " - f"and '2.7'." + f"{implementation!r}. Valid value is '2.7'." ) def fingerprint(self, request: Request) -> bytes: diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index e6d1abe3f..f6bc9ba6f 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,23 +1,15 @@ import json import unittest -import warnings from hashlib import sha1 -from typing import Dict, Mapping, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union from weakref import WeakKeyDictionary -import pytest -from w3lib.url import canonicalize_url - from scrapy.http import Request -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.python import to_bytes from scrapy.utils.request import ( - _deprecated_fingerprint_cache, _fingerprint_cache, - _request_fingerprint_as_bytes, fingerprint, request_authenticate, - request_fingerprint, request_httprepr, request_to_curl, ) @@ -233,168 +225,6 @@ class FingerprintTest(unittest.TestCase): self.assertEqual(actual, expected) -class RequestFingerprintTest(FingerprintTest): - function = staticmethod(request_fingerprint) - cache = _deprecated_fingerprint_cache - known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = ( - ( - Request("http://example.org"), - "b2e5245ef826fd9576c93bd6e392fce3133fab62", - {}, - ), - ( - Request("https://example.org"), - "bd10a0a89ea32cdee77917320f1309b0da87e892", - {}, - ), - ( - Request("https://example.org?a"), - "2fb7d48ae02f04b749f40caa969c0bc3c43204ce", - {}, - ), - ( - Request("https://example.org?a=b"), - "42e5fe149b147476e3f67ad0670c57b4cc57856a", - {}, - ), - ( - Request("https://example.org?a=b&a"), - "d23a9787cb56c6375c2cae4453c5a8c634526942", - {}, - ), - ( - Request("https://example.org?a=b&a=c"), - "9a18a7a8552a9182b7f1e05d33876409e421e5c5", - {}, - ), - ( - Request("https://example.org", method="POST"), - "ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6", - {}, - ), - ( - Request("https://example.org", body=b"a"), - "4bb136e54e715a4ea7a9dd1101831765d33f2d60", - {}, - ), - ( - Request("https://example.org", method="POST", body=b"a"), - "6c6595374a304b293be762f7b7be3f54e9947c65", - {}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "bd10a0a89ea32cdee77917320f1309b0da87e892", - {}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "515b633cb3ca502a33a9d8c890e889ec1e425e65", - {"include_headers": ["A"]}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "505c96e7da675920dfef58725e8c957dfdb38f47", - {"keep_fragments": True}, - ), - ( - Request("https://example.org#a", headers={"A": b"B"}), - "d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da", - {"include_headers": ["A"], "keep_fragments": True}, - ), - ( - Request("https://example.org/ab"), - "4e2870fee58582d6f81755e9b8fdefe3cba0c951", - {}, - ), - ( - Request("https://example.org/a", body=b"b"), - "4e2870fee58582d6f81755e9b8fdefe3cba0c951", - {}, - ), - ) - - def setUp(self) -> None: - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - - def tearDown(self) -> None: - warnings.simplefilter("default", ScrapyDeprecationWarning) - - @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) - def test_part_separation(self): - super().test_part_separation() - - -class RequestFingerprintDeprecationTest(unittest.TestCase): - def test_deprecation_default_parameters(self): - with pytest.warns(ScrapyDeprecationWarning) as warnings: - request_fingerprint(Request("http://www.example.com")) - messages = [str(warning.message) for warning in warnings] - self.assertTrue( - any("Call to deprecated function" in message for message in messages) - ) - self.assertFalse(any("non-default" in message for message in messages)) - - def test_deprecation_non_default_parameters(self): - with pytest.warns(ScrapyDeprecationWarning) as warnings: - request_fingerprint(Request("http://www.example.com"), keep_fragments=True) - messages = [str(warning.message) for warning in warnings] - self.assertTrue( - any("Call to deprecated function" in message for message in messages) - ) - self.assertTrue(any("non-default" in message for message in messages)) - - -class RequestFingerprintAsBytesTest(FingerprintTest): - function = staticmethod(_request_fingerprint_as_bytes) - cache = _deprecated_fingerprint_cache - known_hashes = RequestFingerprintTest.known_hashes - - def test_caching(self): - r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199") - self.assertEqual( - self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key]) - ) - - @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True) - def test_part_separation(self): - super().test_part_separation() - - def test_hashes(self): - actual = [ - self.function(request, **kwargs) for request, _, kwargs in self.known_hashes - ] - expected = [ - bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes - ] - self.assertEqual(actual, expected) - - -_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary() - - -def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False): - if include_headers: - include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers)) - cache = _fingerprint_cache_2_6.setdefault(request, {}) - cache_key = (include_headers, keep_fragments) - if cache_key not in cache: - fp = sha1() - fp.update(to_bytes(request.method)) - fp.update( - to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)) - ) - fp.update(request.body or b"") - if include_headers: - for hdr in include_headers: - if hdr in request.headers: - fp.update(hdr) - for v in request.headers.getlist(hdr): - fp.update(v) - cache[cache_key] = fp.hexdigest() - return cache[cache_key] - - REQUEST_OBJECTS_TO_TEST = ( Request("http://www.example.com/"), Request("http://www.example.com/query?id=111&cat=222"), @@ -424,105 +254,17 @@ REQUEST_OBJECTS_TO_TEST = ( ) -class BackwardCompatibilityTestCase(unittest.TestCase): - def test_function_backward_compatibility(self): - include_headers_to_test = ( - None, - ["Accept-Language"], - ["accept-language", "sessionid"], - ["SESSIONID", "Accept-Language"], - ) - for request_object in REQUEST_OBJECTS_TO_TEST: - for include_headers in include_headers_to_test: - for keep_fragments in (False, True): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - fp = request_fingerprint( - request_object, - include_headers=include_headers, - keep_fragments=keep_fragments, - ) - old_fp = request_fingerprint_2_6( - request_object, - include_headers=include_headers, - keep_fragments=keep_fragments, - ) - self.assertEqual(fp, old_fp) - - def test_component_backward_compatibility(self): - for request_object in REQUEST_OBJECTS_TO_TEST: - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - crawler = get_crawler(prevent_warnings=False) - fp = crawler.request_fingerprinter.fingerprint(request_object) - old_fp = request_fingerprint_2_6(request_object) - self.assertEqual(fp.hex(), old_fp) - - def test_custom_component_backward_compatibility(self): - """Tests that the backward-compatible request fingerprinting class featured - in the documentation is indeed backward compatible and does not cause a - warning to be logged.""" - - class RequestFingerprinter: - cache = WeakKeyDictionary() - - def fingerprint(self, request): - if request not in self.cache: - fp = sha1() - fp.update(to_bytes(request.method)) - fp.update(to_bytes(canonicalize_url(request.url))) - fp.update(request.body or b"") - self.cache[request] = fp.digest() - return self.cache[request] - - for request_object in REQUEST_OBJECTS_TO_TEST: - with warnings.catch_warnings() as logged_warnings: - settings = { - "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, - } - crawler = get_crawler(settings_dict=settings) - fp = crawler.request_fingerprinter.fingerprint(request_object) - old_fp = request_fingerprint_2_6(request_object) - self.assertEqual(fp.hex(), old_fp) - self.assertFalse(logged_warnings) - - class RequestFingerprinterTestCase(unittest.TestCase): def test_default_implementation(self): - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(prevent_warnings=False) - request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - _request_fingerprint_as_bytes(request), - ) - self.assertTrue(logged_warnings) - - def test_deprecated_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6", - } - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) - request = Request("https://example.com") - self.assertEqual( - crawler.request_fingerprinter.fingerprint(request), - _request_fingerprint_as_bytes(request), - ) - self.assertTrue(logged_warnings) - - def test_recommended_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), fingerprint(request), ) - self.assertFalse(logged_warnings) def test_unknown_implementation(self): settings = { From 019443dd5761afd5b4f7bba5948508554f1cb5f1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 10:08:07 -0300 Subject: [PATCH 16/21] Remove settings from default implementation test --- tests/test_utils_request.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index f6bc9ba6f..68f6eb045 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -256,10 +256,7 @@ REQUEST_OBJECTS_TO_TEST = ( class RequestFingerprinterTestCase(unittest.TestCase): def test_default_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", - } - crawler = get_crawler(settings_dict=settings) + crawler = get_crawler() request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), From bacaf0db7ac8b3b424af41d24e12e89a3a15b004 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 10:14:48 -0300 Subject: [PATCH 17/21] Update documentation --- docs/topics/request-response.rst | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 9d64eee45..6dbcb4584 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -477,20 +477,12 @@ REQUEST_FINGERPRINTER_IMPLEMENTATION .. versionadded:: 2.7 -Default: ``'2.6'`` +Default: ``'2.7'`` Determines which request fingerprinting algorithm is used by the default request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). -Possible values are: - -- ``'2.6'`` (default) - - This implementation uses the same request fingerprinting algorithm as - Scrapy 2.6 and earlier versions. - - Even though this is the default value for backward compatibility reasons, - it is a deprecated value. +Possible value is: - ``'2.7'`` @@ -500,29 +492,13 @@ Possible values are: New projects should use this value. The :command:`startproject` command sets this value in the generated ``settings.py`` file. -If you are using the default value (``'2.6'``) for this setting, and you are -using Scrapy components where changing the request fingerprinting algorithm -would cause undesired results, you need to carefully decide when to change the -value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS` -setting to a custom request fingerprinter class that implements the 2.6 request -fingerprinting algorithm and does not log this warning ( -:ref:`2.6-request-fingerprinter` includes an example implementation of such a -class). - Scenarios where changing the request fingerprinting algorithm may cause undesired results include, for example, using the HTTP cache middleware (see :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). Changing the request fingerprinting algorithm would invalidate the current cache, requiring you to redownload all requests again. -Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in -your settings to switch already to the request fingerprinting implementation -that will be the only request fingerprinting implementation available in a -future version of Scrapy, and remove the deprecation warning triggered by using -the default value (``'2.6'``). - -.. _2.6-request-fingerprinter: .. _custom-request-fingerprinter: Writing your own request fingerprinter From 24634f1bb236f72a1a7b73900f8e3b524f7717b5 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 12:29:43 -0300 Subject: [PATCH 18/21] Attend PR comments --- docs/topics/request-response.rst | 30 ------------------- scrapy/settings/default_settings.py | 2 +- .../templates/project/module/settings.py.tmpl | 1 - scrapy/utils/request.py | 16 ++++++++-- scrapy/utils/test.py | 3 -- tests/test_utils_request.py | 14 +++++++++ 6 files changed, 28 insertions(+), 38 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 6dbcb4584..67fc0c6e9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -469,36 +469,6 @@ import path. .. autoclass:: scrapy.utils.request.RequestFingerprinter - -.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION - -REQUEST_FINGERPRINTER_IMPLEMENTATION -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 2.7 - -Default: ``'2.7'`` - -Determines which request fingerprinting algorithm is used by the default -request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). - -Possible value is: - -- ``'2.7'`` - - This implementation was introduced in Scrapy 2.7 to fix an issue of the - previous implementation. - - New projects should use this value. The :command:`startproject` command - sets this value in the generated ``settings.py`` file. - -Scenarios where changing the request fingerprinting algorithm may cause -undesired results include, for example, using the HTTP cache middleware (see -:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). -Changing the request fingerprinting algorithm would invalidate the current -cache, requiring you to redownload all requests again. - - .. _custom-request-fingerprinter: Writing your own request fingerprinter diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 02494bad0..49ab1b5ef 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -260,7 +260,7 @@ REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" +REQUEST_FINGERPRINTER_IMPLEMENTATION = "SENTINEL" RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index ecb1e5e5c..b4779e555 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -88,6 +88,5 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index b230cd214..068e5bdcb 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -5,6 +5,7 @@ scrapy.http.Request objects import hashlib import json +import warnings from typing import ( TYPE_CHECKING, Any, @@ -25,6 +26,7 @@ from w3lib.http import basic_auth_header from w3lib.url import canonicalize_url from scrapy import Request, Spider +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -139,14 +141,22 @@ class RequestFingerprinter: "REQUEST_FINGERPRINTER_IMPLEMENTATION" ) else: - implementation = "2.7" - if implementation == "2.7": + implementation = "SENTINEL" + + if implementation == "SENTINEL": + self._fingerprint = fingerprint + elif implementation == "2.7": + message = ( + "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n" + "And it will be removed in future version of Scrapy." + ) + warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) self._fingerprint = fingerprint else: raise ValueError( f"Got an invalid value on setting " f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': " - f"{implementation!r}. Valid value is '2.7'." + f"{implementation!r}. Valid values are '2.7' and 'SENTINEL'." ) def fingerprint(self, request: Request) -> bytes: diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 709e0b00d..c6a31cacf 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -76,7 +76,6 @@ class TestSpider(Spider): def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, - prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -86,8 +85,6 @@ def get_crawler( # Set by default settings that prevent deprecation warnings. settings: Dict[str, Any] = {} - if prevent_warnings: - settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 68f6eb045..c0c44875e 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,5 +1,6 @@ import json import unittest +import warnings from hashlib import sha1 from typing import Dict, Optional, Tuple, Union from weakref import WeakKeyDictionary @@ -263,6 +264,19 @@ class RequestFingerprinterTestCase(unittest.TestCase): fingerprint(request), ) + def test_deprecated_implementation(self): + settings = { + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } + with warnings.catch_warnings(record=True) as logged_warnings: + crawler = get_crawler(settings_dict=settings) + request = Request("https://example.com") + self.assertEqual( + crawler.request_fingerprinter.fingerprint(request), + fingerprint(request), + ) + self.assertTrue(logged_warnings) + def test_unknown_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5", From 7001193c802029612542ab7a30fa8c0e147a1894 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 12:53:08 -0300 Subject: [PATCH 19/21] Simplify the logic --- scrapy/utils/request.py | 12 ++---------- scrapy/utils/test.py | 3 +++ tests/test_utils_request.py | 7 ------- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 068e5bdcb..db0b44cf4 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -143,21 +143,13 @@ class RequestFingerprinter: else: implementation = "SENTINEL" - if implementation == "SENTINEL": - self._fingerprint = fingerprint - elif implementation == "2.7": + if implementation != "SENTINEL": message = ( "'REQUEST_FINGERPRINTER_IMPLEMENTATION' is a deprecated setting.\n" "And it will be removed in future version of Scrapy." ) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) - self._fingerprint = fingerprint - else: - raise ValueError( - f"Got an invalid value on setting " - f"'REQUEST_FINGERPRINTER_IMPLEMENTATION': " - f"{implementation!r}. Valid values are '2.7' and 'SENTINEL'." - ) + self._fingerprint = fingerprint def fingerprint(self, request: Request) -> bytes: return self._fingerprint(request) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index c6a31cacf..9234ec2ea 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -76,6 +76,7 @@ class TestSpider(Spider): def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, + prevent_warnings: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -85,6 +86,8 @@ def get_crawler( # Set by default settings that prevent deprecation warnings. settings: Dict[str, Any] = {} + if prevent_warnings: + pass settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index c0c44875e..633077eec 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -277,13 +277,6 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertTrue(logged_warnings) - def test_unknown_implementation(self): - settings = { - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5", - } - with self.assertRaises(ValueError): - get_crawler(settings_dict=settings) - class CustomRequestFingerprinterTestCase(unittest.TestCase): def test_include_headers(self): From 53ccf0016d99e54adec6f98236cce37ede835f63 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 31 Jan 2024 13:18:10 -0300 Subject: [PATCH 20/21] Remove empty statement --- scrapy/utils/test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 9234ec2ea..7a8c5c859 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -86,8 +86,6 @@ def get_crawler( # Set by default settings that prevent deprecation warnings. settings: Dict[str, Any] = {} - if prevent_warnings: - pass settings.update(settings_dict or {}) runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or TestSpider) From 1c9d308accd38a91ffa92e3aff8912cd792070eb Mon Sep 17 00:00:00 2001 From: Andy <128531452+Andy-W-Developer@users.noreply.github.com> Date: Tue, 6 Feb 2024 00:52:01 +1300 Subject: [PATCH 21/21] Cover the deprecation and removal of response_httprepr in the release notes (#6216) --- docs/news.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 65d9c5181..d90e32560 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -62,6 +62,9 @@ Deprecation removals 1.0.0, use :attr:`CrawlerRunner.spider_loader ` instead. (:issue:`6010`) +- The :func:`scrapy.utils.response.response_httprepr` function, deprecated in + Scrapy 2.6.0, has now been removed. (:issue:`6111`) + Deprecations ~~~~~~~~~~~~ @@ -1157,6 +1160,9 @@ Deprecations Instead, call :meth:`~scrapy.core.engine.ExecutionEngine.open_spider` first to set the :class:`~scrapy.Spider` object. +- :func:`scrapy.utils.response.response_httprepr` is now deprecated. + (:issue:`4972`) + New features ~~~~~~~~~~~~