mirror of https://github.com/scrapy/scrapy.git
Add content-based image filtering example (#4954)
This commit is contained in:
parent
f5a62a293f
commit
fb3455304d
|
|
@ -774,4 +774,28 @@ To enable your custom media pipeline component you must add its class import pat
|
||||||
|
|
||||||
ITEM_PIPELINES = {"myproject.pipelines.MyImagesPipeline": 300}
|
ITEM_PIPELINES = {"myproject.pipelines.MyImagesPipeline": 300}
|
||||||
|
|
||||||
|
Content-based image filtering pipeline
|
||||||
|
--------------------------------------
|
||||||
|
|
||||||
|
This example overrides ``get_images()`` to filter images using a classifier,
|
||||||
|
such as a TensorFlow_ model. Override ``is_valid_image()`` with your
|
||||||
|
classification logic:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
from scrapy.pipelines.images import ImagesPipeline, ImageException
|
||||||
|
|
||||||
|
|
||||||
|
class ImageClassifierPipeline(ImagesPipeline):
|
||||||
|
def is_valid_image(self, image):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def get_images(self, response, request, info, *, item=None):
|
||||||
|
for path, image, buf in super().get_images(response, request, info, item=item):
|
||||||
|
if not self.is_valid_image(image):
|
||||||
|
raise ImageException("Image does not match criteria")
|
||||||
|
yield path, image, buf
|
||||||
|
|
||||||
|
|
||||||
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5
|
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5
|
||||||
|
.. _TensorFlow: https://tensorflow.org
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue