From c35a7519c0ebc18fe79da0d0e50940334ec4870a Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 8 Aug 2011 13:23:45 -0300 Subject: [PATCH] Correctly handle query parameters on s3:// urls --- scrapy/core/downloader/handlers/s3.py | 6 ++++-- scrapy/tests/test_urlparse_monkeypatches.py | 8 ++++++-- scrapy/xlib/urlparse_monkeypatches.py | 4 +++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 9b89a7936..b9c877a9d 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -27,8 +27,10 @@ class S3DownloadHandler(object): def download_request(self, request, spider): p = urlparse_cached(request) scheme = 'https' if request.meta.get('is_secure') else 'http' - url = '%s://%s.s3.amazonaws.com%s' % (scheme, p.hostname, p.path) + bucket = p.hostname + path = p.path + '?' + p.query if p.query else p.path + url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path) httpreq = request.replace(url=url) self.conn.add_aws_auth_header(httpreq.headers, httpreq.method, \ - '%s/%s' % (p.hostname, p.path)) + '%s/%s' % (bucket, path)) return self._download_http(httpreq, spider) diff --git a/scrapy/tests/test_urlparse_monkeypatches.py b/scrapy/tests/test_urlparse_monkeypatches.py index 7dfca7162..91d75789c 100644 --- a/scrapy/tests/test_urlparse_monkeypatches.py +++ b/scrapy/tests/test_urlparse_monkeypatches.py @@ -4,5 +4,9 @@ import unittest class UrlparseTestCase(unittest.TestCase): - def test_s3_netloc(self): - self.assertEqual(urlparse('s3://bucket/key').netloc, 'bucket') + def test_s3_url(self): + p = urlparse('s3://bucket/key/name?param=value') + self.assertEquals(p.scheme, 's3') + self.assertEquals(p.hostname, 'bucket') + self.assertEquals(p.path, '/key/name') + self.assertEquals(p.query, 'param=value') diff --git a/scrapy/xlib/urlparse_monkeypatches.py b/scrapy/xlib/urlparse_monkeypatches.py index 010a8bb01..5228b6551 100644 --- a/scrapy/xlib/urlparse_monkeypatches.py +++ b/scrapy/xlib/urlparse_monkeypatches.py @@ -1,5 +1,7 @@ -from urlparse import urlparse, uses_netloc +from urlparse import urlparse, uses_netloc, uses_query # workaround for http://bugs.python.org/issue7904 if urlparse('s3://bucket/key').netloc != 'bucket': uses_netloc.append('s3') +if urlparse('s3://bucket/key?key=value').query != 'key=value': + uses_query.append('s3')