From 6d33e12c8b4e0ceb1da325f3e8d3a6308dbadfda Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Tue, 23 Dec 2008 14:31:58 +0000 Subject: [PATCH] aws: sign requests if url hostname ends with s3.amazonaws.com common bucket suffix. This allows a spider to send requests to S3 with out need to sign it inside spider code --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40542 --- scrapy/trunk/scrapy/contrib/aws.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/trunk/scrapy/contrib/aws.py b/scrapy/trunk/scrapy/contrib/aws.py index 0ad99f5bd..22ae23a27 100644 --- a/scrapy/trunk/scrapy/contrib/aws.py +++ b/scrapy/trunk/scrapy/contrib/aws.py @@ -81,6 +81,7 @@ class AWSMiddleware(object): self.secret_key = settings['AWS_SECRET_ACCESS_KEY'] def process_request(self, request, spider): - if spider.domain_name in 's3.amazonaws.com': + if spider.domain_name == 's3.amazonaws.com' \ + or request.url.hostname.endswith('s3.amazonaws.com'): request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) sign_request(request, self.access_key, self.secret_key)