s3: sign requests in downloder middleware to avoid sending signed requests dated more than 15minutes ago

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40469

scrapy/trunk/scrapy/contrib/aws.py

@@ -4,6 +4,7 @@ import hmac
 import base64
 import hashlib
 from urlparse import urlsplit
+from scrapy.conf import settings
 
 
 METADATA_PREFIX = 'x-amz-meta-'
@@ -74,3 +75,12 @@ def sign_request(req, accesskey, secretkey):
     req.headers['Authorization'] = "AWS %s:%s" % (accesskey, b64_hmac)
 
 
+class AWSMiddleware(object):
+    def __init__(self):
+        self.access_key = settings['AWS_ACCESS_KEY_ID']
+        self.secret_key = settings['AWS_SECRET_ACCESS_KEY']
+
+    def process_request(self, request, spider):
+        if spider.domain_name in 's3.amazonaws.com':
+            request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
+            sign_request(request, self.access_key, self.secret_key)

scrapy/trunk/scrapy/contrib/pipeline/s3images.py

@@ -12,7 +12,7 @@ from scrapy.http import Request
 from scrapy.stats import stats
 from scrapy.core.exceptions import DropItem, NotConfigured, HttpException
 from scrapy.contrib.pipeline.media import MediaPipeline
-from scrapy.contrib.aws import canonical_string, sign_request
+from scrapy.contrib.aws import sign_request
 from scrapy.conf import settings
 
 from .images import BaseImagesPipeline, NoimagesDrop, ImageException
@@ -36,6 +36,10 @@ class S3ImagesPipeline(BaseImagesPipeline):
 #             ('270', (270, 270))
     )
 
+    # Automatically sign requests with AWS authorization header,
+    # alternative we can do this using scrapy.contrib.aws.AWSMiddleware
+    sign_requests = True
+
     def __init__(self):
         if not settings['S3_IMAGES']:
             raise NotConfigured
@@ -50,7 +54,8 @@ class S3ImagesPipeline(BaseImagesPipeline):
     def s3request(self, key, method, body=None, headers=None):
         url = 'http://%s.s3.amazonaws.com/%s%s' % (self.bucket_name, self.prefix, key)
         req = Request(url, method=method, body=body, headers=headers)
-        sign_request(req, self.access_key, self.secret_key)
+        if self.sign_requests:
+            sign_request(req, self.access_key, self.secret_key)
         return req
 
     def image_downloaded(self, response, request, info):