From 13735bcf34b0caf60a52e95a907ef390324fdddc Mon Sep 17 00:00:00 2001 From: OmarFarrag Date: Mon, 16 Sep 2019 14:04:06 +0200 Subject: [PATCH] Disallow media extensions unregistered with IANA (#3954) Co-Authored-By: s-sanjay --- scrapy/pipelines/files.py | 9 +++++++++ tests/test_pipeline_files.py | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ea06d2ae8..cc3d10b63 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib +import mimetypes import os import os.path import time @@ -14,6 +15,7 @@ from six.moves.urllib.parse import urlparse from collections import defaultdict import six + try: from cStringIO import StringIO as BytesIO except ImportError: @@ -473,4 +475,11 @@ class FilesPipeline(MediaPipeline): def file_path(self, request, response=None, info=None): media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() media_ext = os.path.splitext(request.url)[1] + # Handles empty and wild extensions by trying to guess the + # mime type then extension or default to empty string otherwise + if media_ext not in mimetypes.types_map: + media_ext = '' + media_type = mimetypes.guess_type(request.url)[0] + if media_type: + media_ext = mimetypes.guess_extension(media_type) return 'full/%s%s' % (media_guid, media_ext) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 0c5aaaa44..cb8f8da18 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -57,6 +57,12 @@ class FilesPipelineTestCase(unittest.TestCase): response=Response("http://www.dorma.co.uk/images/product_details/2532"), info=object()), 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")), + 'full/76c00cef2ef669ae65052661f68d451162829507') + self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ + //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")), + 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png') + def test_fs_store(self): assert isinstance(self.pipeline.store, FSFilesStore)