Cleanup leftover boto2 code in S3DownloaderHandler (#5209)

S3DownloaderHandler.conn is a leftover attribute from 5e99758.
This commit is contained in:
laggardkernel 2021-07-27 17:11:32 +08:00 committed by GitHub
parent b22a004398
commit abe0b37d30
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 13 deletions

View File

@ -1,5 +1,3 @@
from urllib.parse import unquote
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore_available
@ -59,7 +57,7 @@ class S3DownloadHandler:
url = f'{scheme}://{bucket}.s3.amazonaws.com{path}'
if self.anon:
request = request.replace(url=url)
elif self._signer is not None:
else:
import botocore.awsrequest
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
@ -69,14 +67,4 @@ class S3DownloadHandler:
self._signer.add_auth(awsrequest)
request = request.replace(
url=url, headers=awsrequest.headers.items())
else:
signed_headers = self.conn.make_request(
method=request.method,
bucket=bucket,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body,
)
request = request.replace(url=url, headers=signed_headers)
return self._download_http(request, spider)