Merge branch 'master' into typing-crawler

docs/topics/item-pipeline.rst

@@ -215,7 +215,7 @@ item.
             screenshot_url = self.SPLASH_URL.format(encoded_item_url)
             request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
             response = await maybe_deferred_to_future(
-                spider.crawler.engine.download(request, spider)
+                spider.crawler.engine.download(request)
             )
 
             if response.status != 200:

scrapy/crawler.py

@@ -209,16 +209,6 @@ class CrawlerRunner:
         self._active: Set[Deferred] = set()
         self.bootstrap_failed = False
 
-    @property
-    def spiders(self):
-        warnings.warn(
-            "CrawlerRunner.spiders attribute is renamed to "
-            "CrawlerRunner.spider_loader.",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        return self.spider_loader
-
     def crawl(
         self,
         crawler_or_spidercls: Union[Type[Spider], str, Crawler],

scrapy/exporters.py

@@ -7,13 +7,11 @@ import io
 import marshal
 import pickle
 import pprint
-import warnings
 from collections.abc import Mapping
 from xml.sax.saxutils import XMLGenerator
 
 from itemadapter import ItemAdapter, is_item
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import Item
 from scrapy.utils.python import is_listlike, to_bytes, to_unicode
 from scrapy.utils.serialize import ScrapyJSONEncoder
@@ -330,13 +328,7 @@ class PythonItemExporter(BaseItemExporter):
     """
 
     def _configure(self, options, dont_fail=False):
-        self.binary = options.pop("binary", True)
         super()._configure(options, dont_fail)
-        if self.binary:
-            warnings.warn(
-                "PythonItemExporter will drop support for binary export in the future",
-                ScrapyDeprecationWarning,
-            )
         if not self.encoding:
             self.encoding = "utf-8"
 
@@ -351,18 +343,14 @@ class PythonItemExporter(BaseItemExporter):
             return dict(self._serialize_item(value))
         if is_listlike(value):
             return [self._serialize_value(v) for v in value]
-        encode_func = to_bytes if self.binary else to_unicode
         if isinstance(value, (str, bytes)):
-            return encode_func(value, encoding=self.encoding)
+            return to_unicode(value, encoding=self.encoding)
         return value
 
     def _serialize_item(self, item):
         for key, value in ItemAdapter(item).items():
-            key = to_bytes(key) if self.binary else key
             yield key, self._serialize_value(value)
 
     def export_item(self, item):
         result = dict(self._get_serialized_fields(item))
-        if self.binary:
-            result = dict(self._serialize_item(result))
         return result

tests/test_crawler.py

@@ -20,7 +20,6 @@ from scrapy.extensions.throttle import AutoThrottle
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
-from scrapy.utils.misc import load_object
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer, get_mockserver_env
@@ -182,16 +181,6 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
         runner = CrawlerRunner()
         self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
-    def test_deprecated_attribute_spiders(self):
-        with warnings.catch_warnings(record=True) as w:
-            runner = CrawlerRunner(Settings())
-            spiders = runner.spiders
-            self.assertEqual(len(w), 1)
-            self.assertIn("CrawlerRunner.spiders", str(w[0].message))
-            self.assertIn("CrawlerRunner.spider_loader", str(w[0].message))
-            sl_cls = load_object(runner.settings["SPIDER_LOADER_CLASS"])
-            self.assertIsInstance(spiders, sl_cls)
-
 
 class CrawlerProcessTest(BaseCrawlerTest):
     def test_crawler_process_accepts_dict(self):

tests/test_exporters.py

@@ -7,12 +7,10 @@ import tempfile
 import unittest
 from datetime import datetime
 from io import BytesIO
-from warnings import catch_warnings, filterwarnings
 
 import lxml.etree
 from itemadapter import ItemAdapter
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.exporters import (
     BaseItemExporter,
     CsvItemExporter,
@@ -143,7 +141,7 @@ class BaseItemExporterDataclassTest(BaseItemExporterTest):
 
 class PythonItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
-        return PythonItemExporter(binary=False, **kwargs)
+        return PythonItemExporter(**kwargs)
 
     def test_invalid_option(self):
         with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"):
@@ -198,14 +196,6 @@ class PythonItemExporterTest(BaseItemExporterTest):
         self.assertEqual(type(exported["age"][0]), dict)
         self.assertEqual(type(exported["age"][0]["age"][0]), dict)
 
-    def test_export_binary(self):
-        with catch_warnings():
-            filterwarnings("ignore", category=ScrapyDeprecationWarning)
-            exporter = PythonItemExporter(binary=True)
-            value = self.item_class(name="John\xa3", age="22")
-            expected = {b"name": b"John\xc2\xa3", b"age": b"22"}
-            self.assertEqual(expected, exporter.export_item(value))
-
     def test_nonstring_types_item(self):
         item = self._get_nonstring_types_item()
         ie = self._get_exporter()