Correctly handle query parameters on s3:// urls

scrapy/core/downloader/handlers/s3.py

@@ -27,8 +27,10 @@ class S3DownloadHandler(object):
     def download_request(self, request, spider):
         p = urlparse_cached(request)
         scheme = 'https' if request.meta.get('is_secure') else 'http'
-        url = '%s://%s.s3.amazonaws.com%s' % (scheme, p.hostname, p.path)
+        bucket = p.hostname
+        path = p.path + '?' + p.query if p.query else p.path
+        url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
         httpreq = request.replace(url=url)
         self.conn.add_aws_auth_header(httpreq.headers, httpreq.method, \
-                '%s/%s' % (p.hostname, p.path))
+                '%s/%s' % (bucket, path))
         return self._download_http(httpreq, spider)

scrapy/tests/test_urlparse_monkeypatches.py

@@ -4,5 +4,9 @@ import unittest
 
 class UrlparseTestCase(unittest.TestCase):
 
-    def test_s3_netloc(self):
-        self.assertEqual(urlparse('s3://bucket/key').netloc, 'bucket')
+    def test_s3_url(self):
+        p = urlparse('s3://bucket/key/name?param=value')
+        self.assertEquals(p.scheme, 's3')
+        self.assertEquals(p.hostname, 'bucket')
+        self.assertEquals(p.path, '/key/name')
+        self.assertEquals(p.query, 'param=value')

scrapy/xlib/urlparse_monkeypatches.py

@@ -1,5 +1,7 @@
-from urlparse import urlparse, uses_netloc
+from urlparse import urlparse, uses_netloc, uses_query
 
 # workaround for http://bugs.python.org/issue7904
 if urlparse('s3://bucket/key').netloc != 'bucket':
     uses_netloc.append('s3')
+if urlparse('s3://bucket/key?key=value').query != 'key=value':
+    uses_query.append('s3')