diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index b542c6c05..c67d2d627 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -774,4 +774,28 @@ To enable your custom media pipeline component you must add its class import pat ITEM_PIPELINES = {"myproject.pipelines.MyImagesPipeline": 300} +Content-based image filtering pipeline +-------------------------------------- + +This example overrides ``get_images()`` to filter images using a classifier, +such as a TensorFlow_ model. Override ``is_valid_image()`` with your +classification logic: + +.. code-block:: python + + from scrapy.pipelines.images import ImagesPipeline, ImageException + + + class ImageClassifierPipeline(ImagesPipeline): + def is_valid_image(self, image): + raise NotImplementedError + + def get_images(self, response, request, info, *, item=None): + for path, image, buf in super().get_images(response, request, info, item=item): + if not self.is_valid_image(image): + raise ImageException("Image does not match criteria") + yield path, image, buf + + .. _MD5 hash: https://en.wikipedia.org/wiki/MD5 +.. _TensorFlow: https://tensorflow.org diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index ee391a49e..919f67240 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -125,13 +125,11 @@ def _send_catch_log_deferred( **named, ) d.addErrback(logerror, receiver) - # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html + d2: Deferred[tuple[TypingAny, TypingAny]] = d.addBoth( - lambda result: ( - receiver, # pylint: disable=cell-var-from-loop # noqa: B023 - result, - ) + lambda result, recv: (recv, result), receiver ) + dfds.append(d2) results = yield DeferredList(dfds) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 908bb417c..3282b0591 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -539,6 +539,56 @@ class TestBaseSettings: msg = caplog.records[0].message assert "tests.test_settings.Component1" in msg + def test_getdictorlist(self): + settings = BaseSettings() + + # No value and no default → {} + assert settings.getdictorlist("MISSING") == {} + + # String: valid JSON dict + settings.set("S_DICT_STR", '{"key": "val"}') + assert settings.getdictorlist("S_DICT_STR") == {"key": "val"} + + # String: valid JSON list + settings.set("S_LIST_STR", '["a", "b"]') + assert settings.getdictorlist("S_LIST_STR") == ["a", "b"] + + # String: invalid JSON → comma-split fallback + settings.set("S_CSV", "a,b,c") + assert settings.getdictorlist("S_CSV") == ["a", "b", "c"] + + # String: valid JSON but not dict or list → ValueError caught → comma-split + settings.set("S_JSON_NUMBER", "123") + assert settings.getdictorlist("S_JSON_NUMBER") == ["123"] + + # Tuple → list + settings.set("S_TUPLE", ("x", "y")) + assert settings.getdictorlist("S_TUPLE") == ["x", "y"] + + # Unsupported type → raises ValueError + settings.set("S_INT", 42) + with pytest.raises(ValueError, match="must be a dict, list, tuple, or string"): + settings.getdictorlist("S_INT") + + # Dict value → deepcopy returned + settings.set("S_DICT", {"key": "val"}) + assert settings.getdictorlist("S_DICT") == {"key": "val"} + + # List value → deepcopy returned + settings.set("S_LIST", ["a", "b"]) + assert settings.getdictorlist("S_LIST") == ["a", "b"] + + def test_repr_pretty_(self): + settings = BaseSettings({"key": "value"}) + mock_p = mock.Mock() + + settings._repr_pretty_(mock_p, cycle=False) + assert mock_p.text.call_count == 1 + + mock_p.reset_mock() + settings._repr_pretty_(mock_p, cycle=True) + mock_p.text.assert_called_once_with(repr(settings)) + def test_getwithbase_invalid_setting_name(self): settings = BaseSettings() with pytest.raises( @@ -710,6 +760,15 @@ def test_remove_from_list(before, name, item, after): assert settings.getpriority(name) == expected_settings.getpriority(name) +def test_deprecated_dns_resolver_setting(): + settings = Settings() + with pytest.warns( + ScrapyDeprecationWarning, + match="The DNS_RESOLVER setting is deprecated", + ): + settings.get("DNS_RESOLVER") + + def test_deprecated_concurrent_requests_per_ip_setting(): settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) with pytest.warns(