Disallow media extensions unregistered with IANA (#3954)

Co-Authored-By: s-sanjay <sanjay537@gmail.com>

scrapy/pipelines/files.py

@@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+import mimetypes
 import os
 import os.path
 import time
@@ -14,6 +15,7 @@ from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
 
+
 try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
@@ -473,4 +475,11 @@ class FilesPipeline(MediaPipeline):
     def file_path(self, request, response=None, info=None):
         media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         media_ext = os.path.splitext(request.url)[1]
+        # Handles empty and wild extensions by trying to guess the
+        # mime type then extension or default to empty string otherwise
+        if media_ext not in mimetypes.types_map:
+            media_ext = ''
+            media_type = mimetypes.guess_type(request.url)[0]
+            if media_type:
+                media_ext = mimetypes.guess_extension(media_type)
         return 'full/%s%s' % (media_guid, media_ext)

tests/test_pipeline_files.py

@@ -57,6 +57,12 @@ class FilesPipelineTestCase(unittest.TestCase):
                                    response=Response("http://www.dorma.co.uk/images/product_details/2532"),
                                    info=object()),
                          'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+        self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
+                         'full/76c00cef2ef669ae65052661f68d451162829507')
+        self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
+                                    //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
+                         'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
+                         
 
     def test_fs_store(self):
         assert isinstance(self.pipeline.store, FSFilesStore)