diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 82c0aaa88..f258ff748 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -322,6 +322,19 @@ By default, there are no size constraints, so all images are processed. .. _topics-media-pipeline-override: +Allowing redirections +--------------------- + +.. setting:: MEDIA_ALLOW_REDIRECTS + +By default media pipelines ignore redirects, i.e. an HTTP redirection +to a media file URL request will mean the media download is considered failed. + +To handle media redirections, set this settings to ``True``: + + MEDIA_ALLOW_REDIRECTS = True + + Extending the Media Pipelines ============================= diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index bdc0f24e5..eae03752a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -249,7 +249,7 @@ class FilesPipeline(MediaPipeline): resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD ) - super(FilesPipeline, self).__init__(download_func=download_func) + super(FilesPipeline, self).__init__(download_func=download_func, settings=settings) @classmethod def from_settings(cls, settings): diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 57f70499e..404bbf5bf 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,10 +1,13 @@ from __future__ import print_function +import functools import logging from collections import defaultdict from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure +from scrapy.settings import Settings +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter @@ -24,9 +27,23 @@ class MediaPipeline(object): self.downloaded = {} self.waiting = defaultdict(list) - def __init__(self, download_func=None): + def __init__(self, download_func=None, settings=None): self.download_func = download_func + if isinstance(settings, dict) or settings is None: + settings = Settings(settings) + resolve = functools.partial(self._key_for_pipe, + base_class_name="MediaPipeline", + settings=settings) + self.allow_redirects = settings.getbool( + resolve('MEDIA_ALLOW_REDIRECTS'), False + ) + self._handle_statuses(self.allow_redirects) + + def _handle_statuses(self, allow_redirects): + self.handle_httpstatus_list = None + if allow_redirects: + self.handle_httpstatus_list = SequenceExclude(range(300, 400)) def _key_for_pipe(self, key, base_class_name=None, settings=None): @@ -93,6 +110,12 @@ class MediaPipeline(object): ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _modify_media_request(self, request): + if self.handle_httpstatus_list: + request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list + else: + request.meta['handle_httpstatus_all'] = True + def _check_media_to_download(self, result, request, info): if result is not None: return result @@ -103,7 +126,7 @@ class MediaPipeline(object): callback=self.media_downloaded, callbackArgs=(request, info), errback=self.media_failed, errbackArgs=(request, info)) else: - request.meta['handle_httpstatus_all'] = True + self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), diff --git a/tests/mockserver.py b/tests/mockserver.py index e611cc3ec..26ab51183 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -5,13 +5,17 @@ from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource +from twisted.web.static import File from twisted.web.test.test_webclient import PayloadResource from twisted.web.server import GzipEncoderFactory from twisted.web.resource import EncodingResourceWrapper +from twisted.web.util import redirectTo from twisted.internet import reactor, ssl from twisted.internet.task import deferLater + from scrapy.utils.python import to_bytes, to_unicode +from tests import tests_datadir def getarg(request, name, default=None, type=None): @@ -120,6 +124,16 @@ class Echo(LeafResource): return to_bytes(json.dumps(output)) +class RedirectTo(LeafResource): + + def render(self, request): + goto = getarg(request, b'goto', b'/') + # we force the body content, otherwise Twisted redirectTo() + # returns HTML with