From 45ff6ec28ac44114b1c8b98e151ff3c21cd9b994 Mon Sep 17 00:00:00 2001 From: Lukasz Biedrycki Date: Fri, 16 Aug 2013 18:47:56 +0200 Subject: [PATCH] Test reorganization and new tests for Files and Images Pipelines, PEP8 changes in MediaPipeline --- scrapy/contrib/pipeline/media.py | 5 +- scrapy/tests/test_pipeline_files.py | 108 +++++++++++++++++++++++++++ scrapy/tests/test_pipeline_images.py | 9 --- 3 files changed, 111 insertions(+), 11 deletions(-) create mode 100644 scrapy/tests/test_pipeline_files.py diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 9491836d2..42217fd96 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -7,6 +7,7 @@ from scrapy import log from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter + class MediaPipeline(object): LOG_FAILED_RESULTS = True @@ -65,7 +66,7 @@ class MediaPipeline(object): dfd.addCallback(self._check_media_to_download, request, info) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) dfd.addErrback(log.err, spider=info.spider) - return dfd.addBoth(lambda _: wad) # it must return wad at last + return dfd.addBoth(lambda _: wad) # it must return wad at last def _check_media_to_download(self, result, request, info): if result is not None: @@ -91,7 +92,7 @@ class MediaPipeline(object): result.frames = [] result.stack = None info.downloading.remove(fp) - info.downloaded[fp] = result # cache result + info.downloaded[fp] = result # cache result for wad in info.waiting.pop(fp): defer_result(result).chainDeferred(wad) diff --git a/scrapy/tests/test_pipeline_files.py b/scrapy/tests/test_pipeline_files.py new file mode 100644 index 000000000..c1b59c0f0 --- /dev/null +++ b/scrapy/tests/test_pipeline_files.py @@ -0,0 +1,108 @@ +import mock +import os +import time +from tempfile import mkdtemp +from shutil import rmtree + +from twisted.trial import unittest +from twisted.internet import defer + +from scrapy.contrib.pipeline.files import FilesPipeline, FSFilesStore +from scrapy.item import Item, Field +from scrapy.http import Request, Response + + +def _mocked_download_func(request, info): + response = request.meta.get('response') + return response() if callable(response) else response + + +class FilesPipelineTestCase(unittest.TestCase): + + def setUp(self): + self.tempdir = mkdtemp() + self.pipeline = FilesPipeline(self.tempdir, download_func=_mocked_download_func) + self.pipeline.open_spider(None) + + def tearDown(self): + rmtree(self.tempdir) + + def test_file_path(self): + image_path = self.pipeline.file_key + self.assertEqual(image_path("https://dev.mydeco.com/mydeco.pdf"), + 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') + self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt"), + 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') + self.assertEqual(image_path("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc"), + 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') + self.assertEqual(image_path("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"), + 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532/"), + 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') + self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532"), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + + def test_fs_store(self): + assert isinstance(self.pipeline.store, FSFilesStore) + self.assertEqual(self.pipeline.store.basedir, self.tempdir) + + key = 'some/image/key.jpg' + path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') + self.assertEqual(self.pipeline.store._get_filesystem_path(key), path) + + @defer.inlineCallbacks + def test_file_not_expired(self): + item_url = "http://example.com/file.pdf" + item = _create_item_with_files(item_url) + patchers = [ + mock.patch.object(FilesPipeline, 'inc_stats', return_value=True), + mock.patch.object(FSFilesStore, 'stat_image', return_value={ + 'checksum': 'abc', 'last_modified': time.time()}), + mock.patch.object(FilesPipeline, 'get_media_requests', + return_value=[_prepare_request_object(item_url)]) + ] + map(lambda p: p.start(), patchers) + + result = yield self.pipeline.process_item(item, None) + self.assertEqual(result['files'][0]['checksum'], 'abc') + + map(lambda p: p.stop(), patchers) + + @defer.inlineCallbacks + def test_file_expired(self): + item_url = "http://example.com/file2.pdf" + item = _create_item_with_files(item_url) + patchers = [ + mock.patch.object(FSFilesStore, 'stat_image', return_value={ + 'checksum': 'abc', + 'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}), + mock.patch.object(FilesPipeline, 'get_media_requests', + return_value=[_prepare_request_object(item_url)]), + mock.patch.object(FilesPipeline, 'inc_stats', return_value=True) + ] + map(lambda p: p.start(), patchers) + + result = yield self.pipeline.process_item(item, None) + self.assertNotEqual(result['files'][0]['checksum'], 'abc') + + map(lambda p: p.stop(), patchers) + + +class ItemWithFiles(Item): + file_urls = Field() + files = Field() + + +def _create_item_with_files(*files): + item = ItemWithFiles() + item['file_urls'] = files + return item + + +def _prepare_request_object(item_url): + return Request( + item_url, + meta={'response': Response(item_url, status=200, body='data')}) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_pipeline_images.py b/scrapy/tests/test_pipeline_images.py index 74acac9b3..692e91afa 100644 --- a/scrapy/tests/test_pipeline_images.py +++ b/scrapy/tests/test_pipeline_images.py @@ -61,15 +61,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(thumbnail_name("/tmp/some.name/foo", name), 'thumbs/50/92dac2a6a2072c5695a5dff1f865b3cb70c657bb.jpg') - def test_fs_store(self): - from scrapy.contrib.pipeline.files import FSFilesStore - assert isinstance(self.pipeline.store, FSFilesStore) - self.assertEqual(self.pipeline.store.basedir, self.tempdir) - - key = 'some/image/key.jpg' - path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') - self.assertEqual(self.pipeline.store._get_filesystem_path(key), path) - def test_convert_image(self): SIZE = (100, 100) # straigh forward case: RGB and JPEG