diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index cd84905c5..86550d7a4 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -50,7 +50,7 @@ this: 4. When the files are downloaded, another field (``files``) will be populated with the results. This field will contain a list of dicts with information about the downloaded files, such as the downloaded path, the original - scraped url (taken from the ``file_urls`` field) , and the file checksum. + scraped url (taken from the ``file_urls`` field), the file checksum and the file status. The files in the list of the ``files`` field will retain the same order of the original ``file_urls`` field. If some file failed downloading, an error will be logged and the file won't be present in the ``files`` field. @@ -470,6 +470,14 @@ See here the methods that you can override in your custom Files Pipeline: * ``checksum`` - a `MD5 hash`_ of the image contents + * ``status`` - the file status indication. It can be one of the following: + + * ``downloaded`` - file was downloaded. + * ``uptodate`` - file was not downloaded, as it was downloaded recently, + according to the file expiration policy. + * ``cached`` - file was already scheduled for download, by another item + sharing the same file. + The list of tuples received by :meth:`~item_completed` is guaranteed to retain the same order of the requests returned from the :meth:`~get_media_requests` method. @@ -479,7 +487,8 @@ See here the methods that you can override in your custom Files Pipeline: [(True, {'checksum': '2b00042f7481c7b056c4b410d28f33cf', 'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg', - 'url': 'http://www.example.com/files/product1.pdf'}), + 'url': 'http://www.example.com/files/product1.pdf', + 'status': 'downloaded'}), (False, Failure(...))] diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index cd3e29057..7d86d0d56 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -432,7 +432,7 @@ class FilesPipeline(MediaPipeline): self.inc_stats(info.spider, 'uptodate') checksum = result.get('checksum', None) - return {'url': request.url, 'path': path, 'checksum': checksum} + return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'} path = self.file_path(request, info=info) dfd = defer.maybeDeferred(self.store.stat_file, path, info) @@ -509,7 +509,7 @@ class FilesPipeline(MediaPipeline): ) raise FileException(str(exc)) - return {'url': request.url, 'path': path, 'checksum': checksum} + return {'url': request.url, 'path': path, 'checksum': checksum, 'status': status} def inc_stats(self, spider, status): spider.crawler.stats.inc_value('file_count', spider=spider) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 74263e50c..9af5affec 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -91,6 +91,11 @@ class FileDownloadCrawlTestCase(TestCase): file_dl_success = 'File (downloaded): Downloaded file from' self.assertEqual(logs.count(file_dl_success), 3) + # check that the images/files status is `downloaded` + for item in items: + for i in item[self.media_key]: + self.assertEqual(i['status'], 'downloaded') + # check that the images/files checksums are what we know they should be if self.expected_checksums is not None: checksums = set( diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 34fc8dcb6..6bbcbc2e9 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -93,6 +93,7 @@ class FilesPipelineTestCase(unittest.TestCase): result = yield self.pipeline.process_item(item, None) self.assertEqual(result['files'][0]['checksum'], 'abc') + self.assertEqual(result['files'][0]['status'], 'uptodate') for p in patchers: p.stop() @@ -114,6 +115,29 @@ class FilesPipelineTestCase(unittest.TestCase): result = yield self.pipeline.process_item(item, None) self.assertNotEqual(result['files'][0]['checksum'], 'abc') + self.assertEqual(result['files'][0]['status'], 'downloaded') + + for p in patchers: + p.stop() + + @defer.inlineCallbacks + def test_file_cached(self): + item_url = "http://example.com/file3.pdf" + item = _create_item_with_files(item_url) + patchers = [ + mock.patch.object(FilesPipeline, 'inc_stats', return_value=True), + mock.patch.object(FSFilesStore, 'stat_file', return_value={ + 'checksum': 'abc', + 'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}), + mock.patch.object(FilesPipeline, 'get_media_requests', + return_value=[_prepare_request_object(item_url, flags=['cached'])]) + ] + for p in patchers: + p.start() + + result = yield self.pipeline.process_item(item, None) + self.assertNotEqual(result['files'][0]['checksum'], 'abc') + self.assertEqual(result['files'][0]['status'], 'cached') for p in patchers: p.stop() @@ -412,10 +436,10 @@ def _create_item_with_files(*files): return item -def _prepare_request_object(item_url): +def _prepare_request_object(item_url, flags=None): return Request( item_url, - meta={'response': Response(item_url, status=200, body=b'data')}) + meta={'response': Response(item_url, status=200, body=b'data', flags=flags)}) if __name__ == "__main__":