RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

adjust variable wording and redirect logic

This commit is contained in:
Bernardas 2016-08-24 08:40:46 +00:00 committed by Paul Tremberth
parent f0b4077f81
commit 3cef1cd451
2 changed files with 11 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
scrapy/pipelines/media.py
View File

@ -6,6 +6,7 @@ from collections import defaultdict
from twisted.internet.defer import Deferred, DeferredList from twisted.internet.defer import Deferred, DeferredList
from twisted.python.failure import Failure from twisted.python.failure import Failure
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.settings import Settings from scrapy.settings import Settings
from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.defer import mustbe_deferred, defer_result
from scrapy.utils.request import request_fingerprint from scrapy.utils.request import request_fingerprint
@ -36,7 +37,7 @@ class MediaPipeline(object):
self.allow_redirects = settings.getbool( self.allow_redirects = settings.getbool(
resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS
) )
self.allow_httpstatus_list = settings.getlist( self.handle_httpstatus_list = settings.getlist(
resolve('MEDIA_HTTPSTATUS_LIST'), [] resolve('MEDIA_HTTPSTATUS_LIST'), []
) )
@ -107,17 +108,15 @@ class MediaPipeline(object):
def _modify_media_request(self, request): def _modify_media_request(self, request):
httpstatus_list = [] httpstatus_list = []
if self.allow_httpstatus_list: if self.handle_httpstatus_list:
httpstatus_list = self.allow_httpstatus_list httpstatus_list = self.handle_httpstatus_list
elif self.allow_redirects: if self.allow_redirects:
if not httpstatus_list: if not httpstatus_list:
httpstatus_list = list(range(0, 300)) + list(range(400, 1000)) httpstatus_list = [i for i in range(1000)
if i not in RedirectMiddleware.allowed_status]
else: else:
for i in range(300, 400): httpstatus_list = [i for i in httpstatus_list
try: if i not in RedirectMiddleware.allowed_status]
httpstatus_list.remove(i)
except ValueError:
pass
if httpstatus_list: if httpstatus_list:
request.meta['handle_httpstatus_list'] = httpstatus_list request.meta['handle_httpstatus_list'] = httpstatus_list
else: else:

4
tests/test_pipeline_media.py
View File

@ -89,9 +89,9 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True} assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True}
request = Request('http://url') request = Request('http://url')
self.pipe.allow_httpstatus_list = list(range(100)) self.pipe.handle_httpstatus_list = list(range(100))
assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))} assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))}
self.pipe.allow_httpstatus_list = None self.pipe.handle_httpstatus_list = None
request = Request('http://url') request = Request('http://url')
self.pipe.allow_redirects = True self.pipe.allow_redirects = True