From 0dbb5d44ae8f841d60a34ce507ca54ca8cf49a23 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Fri, 9 Apr 2010 14:16:00 -0300 Subject: [PATCH] images: avoid signing images based on spider name or request hostname, use request.meta instead --- scrapy/contrib/aws.py | 13 +++++-------- scrapy/contrib/pipeline/images.py | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/scrapy/contrib/aws.py b/scrapy/contrib/aws.py index 91b134a0e..b62c9e6b3 100644 --- a/scrapy/contrib/aws.py +++ b/scrapy/contrib/aws.py @@ -5,13 +5,13 @@ because Amazon Web Service use timestamps for authentication. """ import os -import time - -from scrapy.utils.httpobj import urlparse_cached +from time import strftime, gmtime from scrapy.utils.aws import sign_request from scrapy.conf import settings + class AWSMiddleware(object): + def __init__(self): self.access_key = settings['AWS_ACCESS_KEY_ID'] or \ os.environ.get('AWS_ACCESS_KEY_ID') @@ -19,9 +19,6 @@ class AWSMiddleware(object): os.environ.get('AWS_SECRET_ACCESS_KEY') def process_request(self, request, spider): - hostname = urlparse_cached(request).hostname - if spider.name == 's3.amazonaws.com' \ - or (hostname and hostname.endswith('s3.amazonaws.com')): - request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", \ - time.gmtime()) + if request.meta.get('sign_s3_request'): + request.headers['Date'] = strftime("%a, %d %b %Y %H:%M:%S GMT", gmtime()) sign_request(request, self.access_key, self.secret_key) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index a56b3da0b..640cc7179 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -143,7 +143,7 @@ class S3ImagesStore(object): def _build_request(self, key, method, body=None, headers=None): url = 'http://%s.s3.amazonaws.com/%s%s' % (self.bucket, self.prefix, key) return Request(url, method=method, body=body, headers=headers, \ - priority=self.request_priority) + meta={'sign_s3_request': True}, priority=self.request_priority) def _download_request(self, request, info): """This method is used for HEAD and PUT requests sent to amazon S3