Test reorganization and new tests for Files and Images Pipelines, PEP8 changes in MediaPipeline

scrapy/contrib/pipeline/media.py

@@ -7,6 +7,7 @@ from scrapy import log
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 
+
 class MediaPipeline(object):
 
     LOG_FAILED_RESULTS = True
@@ -65,7 +66,7 @@ class MediaPipeline(object):
         dfd.addCallback(self._check_media_to_download, request, info)
         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
         dfd.addErrback(log.err, spider=info.spider)
-        return dfd.addBoth(lambda _: wad) # it must return wad at last
+        return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
     def _check_media_to_download(self, result, request, info):
         if result is not None:
@@ -91,7 +92,7 @@ class MediaPipeline(object):
             result.frames = []
             result.stack = None
         info.downloading.remove(fp)
-        info.downloaded[fp] = result # cache result
+        info.downloaded[fp] = result  # cache result
         for wad in info.waiting.pop(fp):
             defer_result(result).chainDeferred(wad)
 

scrapy/tests/test_pipeline_files.py

@@ -0,0 +1,108 @@
+import mock
+import os
+import time
+from tempfile import mkdtemp
+from shutil import rmtree
+
+from twisted.trial import unittest
+from twisted.internet import defer
+
+from scrapy.contrib.pipeline.files import FilesPipeline, FSFilesStore
+from scrapy.item import Item, Field
+from scrapy.http import Request, Response
+
+
+def _mocked_download_func(request, info):
+    response = request.meta.get('response')
+    return response() if callable(response) else response
+
+
+class FilesPipelineTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = mkdtemp()
+        self.pipeline = FilesPipeline(self.tempdir, download_func=_mocked_download_func)
+        self.pipeline.open_spider(None)
+
+    def tearDown(self):
+        rmtree(self.tempdir)
+
+    def test_file_path(self):
+        image_path = self.pipeline.file_key
+        self.assertEqual(image_path("https://dev.mydeco.com/mydeco.pdf"),
+                         'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
+        self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt"),
+                         'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
+        self.assertEqual(image_path("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc"),
+                         'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
+        self.assertEqual(image_path("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"),
+                         'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
+        self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532/"),
+                         'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
+        self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532"),
+                         'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+
+    def test_fs_store(self):
+        assert isinstance(self.pipeline.store, FSFilesStore)
+        self.assertEqual(self.pipeline.store.basedir, self.tempdir)
+
+        key = 'some/image/key.jpg'
+        path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg')
+        self.assertEqual(self.pipeline.store._get_filesystem_path(key), path)
+
+    @defer.inlineCallbacks
+    def test_file_not_expired(self):
+        item_url = "http://example.com/file.pdf"
+        item = _create_item_with_files(item_url)
+        patchers = [
+            mock.patch.object(FilesPipeline, 'inc_stats', return_value=True),
+            mock.patch.object(FSFilesStore, 'stat_image', return_value={
+                'checksum': 'abc', 'last_modified': time.time()}),
+            mock.patch.object(FilesPipeline, 'get_media_requests',
+                              return_value=[_prepare_request_object(item_url)])
+        ]
+        map(lambda p: p.start(), patchers)
+
+        result = yield self.pipeline.process_item(item, None)
+        self.assertEqual(result['files'][0]['checksum'], 'abc')
+
+        map(lambda p: p.stop(), patchers)
+
+    @defer.inlineCallbacks
+    def test_file_expired(self):
+        item_url = "http://example.com/file2.pdf"
+        item = _create_item_with_files(item_url)
+        patchers = [
+            mock.patch.object(FSFilesStore, 'stat_image', return_value={
+                'checksum': 'abc',
+                'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}),
+            mock.patch.object(FilesPipeline, 'get_media_requests',
+                              return_value=[_prepare_request_object(item_url)]),
+            mock.patch.object(FilesPipeline, 'inc_stats', return_value=True)
+        ]
+        map(lambda p: p.start(), patchers)
+
+        result = yield self.pipeline.process_item(item, None)
+        self.assertNotEqual(result['files'][0]['checksum'], 'abc')
+
+        map(lambda p: p.stop(), patchers)
+
+
+class ItemWithFiles(Item):
+    file_urls = Field()
+    files = Field()
+
+
+def _create_item_with_files(*files):
+    item = ItemWithFiles()
+    item['file_urls'] = files
+    return item
+
+
+def _prepare_request_object(item_url):
+    return Request(
+        item_url,
+        meta={'response': Response(item_url, status=200, body='data')})
+
+if __name__ == "__main__":
+    unittest.main()

scrapy/tests/test_pipeline_images.py

@@ -61,15 +61,6 @@ class ImagesPipelineTestCase(unittest.TestCase):
         self.assertEqual(thumbnail_name("/tmp/some.name/foo", name),
                          'thumbs/50/92dac2a6a2072c5695a5dff1f865b3cb70c657bb.jpg')
 
-    def test_fs_store(self):
-        from scrapy.contrib.pipeline.files import FSFilesStore
-        assert isinstance(self.pipeline.store, FSFilesStore)
-        self.assertEqual(self.pipeline.store.basedir, self.tempdir)
-
-        key = 'some/image/key.jpg'
-        path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg')
-        self.assertEqual(self.pipeline.store._get_filesystem_path(key), path)
-
     def test_convert_image(self):
         SIZE = (100, 100)
         # straigh forward case: RGB and JPEG