From 708f1b009b9b23971d73bfc7bc09163969ab6e00 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 21:36:33 +0100 Subject: [PATCH] Add integration tests for MEDIA_ALLOW_REDIRECTS --- tests/test_pipeline_crawl.py | 163 +++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 tests/test_pipeline_crawl.py diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py new file mode 100644 index 000000000..1f5b80954 --- /dev/null +++ b/tests/test_pipeline_crawl.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +import os +import shutil + +from testfixtures import LogCapture +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from w3lib.url import add_or_replace_parameter + +from scrapy.crawler import CrawlerRunner +from scrapy import signals +from tests.mockserver import MockServer +from tests.spiders import SimpleSpider + + +class MediaDownloadSpider(SimpleSpider): + name = 'mediadownload' + + def _process_url(self, url): + return url + + def parse(self, response): + self.logger.info(response.headers) + self.logger.info(response.text) + item = { + 'images': [], + 'image_urls': [ + self._process_url(response.urljoin(href)) + for href in response.xpath(''' + //table[thead/tr/th="Filename"] + /tbody//a/@href + ''').extract()], + } + yield item + + +class BrokenLinksMediaDownloadSpider(MediaDownloadSpider): + name = 'brokenmedia' + + def _process_url(self, url): + return url + '.foo' + + +class RedirectedMediaDownloadSpider(MediaDownloadSpider): + name = 'redirectedmedia' + + def _process_url(self, url): + return add_or_replace_parameter( + 'http://localhost:8998/redirect-to', + 'goto', url) + + +class MediaDownloadCrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + # prepare a directory for storing files + self.tmpmediastore = self.mktemp() + os.mkdir(self.tmpmediastore) + self.settings = { + 'ITEM_PIPELINES': {'scrapy.pipelines.images.ImagesPipeline': 1}, + 'IMAGES_STORE': self.tmpmediastore, + } + self.runner = CrawlerRunner(self.settings) + self.items = [] + # these are the checksums for images in test_site/files/images + # - scrapy.png + # - python-powered-h-50x65.png + # - python-logo-master-v3-TM-flattened.png + self.expected_checksums = set([ + 'a7020c30837f971084834e603625af58', + 'acac52d42b63cf2c3b05832641f3a53c', + '195672ac5888feb400fbf7b352553afe']) + + def tearDown(self): + shutil.rmtree(self.tmpmediastore) + self.items = [] + self.mockserver.__exit__(None, None, None) + + def _on_item_scraped(self, item): + self.items.append(item) + + def _create_crawler(self, spider_class): + crawler = self.runner.create_crawler(spider_class) + crawler.signals.connect(self._on_item_scraped, signals.item_scraped) + return crawler + + def _assert_files_downloaded(self, items, logs): + self.assertEqual(len(items), 1) + self.assertIn('images', items[0]) + + # check that logs show the expected number of successful file downloads + file_dl_success = 'File (downloaded): Downloaded file from' + self.assertEqual(logs.count(file_dl_success), 3) + + # check that the images checksums are what we know they should be + checksums = set( + i['checksum'] + for item in items + for i in item['images']) + self.assertEqual(checksums, self.expected_checksums) + + # check that the image files where actually written to the media store + for item in items: + for i in item['images']: + self.assertTrue( + os.path.exists( + os.path.join(self.tmpmediastore, i['path']))) + + def _assert_files_download_failure(self, crawler, items, code, logs): + + # check that the item does NOT have the "images" field populated + self.assertEqual(len(items), 1) + self.assertIn('images', items[0]) + self.assertFalse(items[0]['images']) + + # check that there was 1 successful fetch and 3 other responses with non-200 code + self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) + self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3) + + # check that logs do show the failure on the file downloads + file_dl_failure = 'File (code: %d): Error downloading file from' % code + self.assertEqual(logs.count(file_dl_failure), 3) + + # check that no files were written to the media store + self.assertEqual(os.listdir(self.tmpmediastore), []) + + @defer.inlineCallbacks + def test_download_media(self): + crawler = self._create_crawler(MediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_downloaded(self.items, str(log)) + + @defer.inlineCallbacks + def test_download_media_wrong_urls(self): + crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_download_failure(crawler, self.items, 404, str(log)) + + @defer.inlineCallbacks + def test_download_media_redirected_default_failure(self): + crawler = self._create_crawler(RedirectedMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_download_failure(crawler, self.items, 302, str(log)) + + @defer.inlineCallbacks + def test_download_media_redirected_allowed(self): + settings = dict(self.settings) + settings.update({'MEDIA_ALLOW_REDIRECTS': True}) + self.runner = CrawlerRunner(settings) + + crawler = self._create_crawler(RedirectedMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_downloaded(self.items, str(log)) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)