diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 1398140b4..933bc7e8d 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -39,12 +39,13 @@ from twisted.internet.protocol import Protocol, ClientCreator from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes class ReceivedDataProtocol(Protocol): def __init__(self, filename=None): self.__filename = filename - self.body = open(filename, "w") if filename else BytesIO() + self.body = open(filename, "wb") if filename else BytesIO() self.size = 0 def dataReceived(self, data): @@ -97,7 +98,7 @@ class FTPDownloadHandler(object): protocol.close() body = protocol.filename or protocol.body.read() headers = {"local filename": protocol.filename or '', "size": protocol.size} - return respcls(url=request.url, status=200, body=body, headers=headers) + return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers) def _failed(self, result, request): message = result.getErrorMessage() @@ -106,6 +107,6 @@ class FTPDownloadHandler(object): if m: ftpcode = m.group() httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"]) - return Response(url=request.url, status=httpcode, body=message) + return Response(url=request.url, status=httpcode, body=to_bytes(message)) raise result.type(result.value) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index ec2947003..313e74ec9 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,13 +1,6 @@ tests/test_linkextractors_deprecated.py tests/test_proxy_connect.py -scrapy/xlib/tx/iweb.py -scrapy/xlib/tx/interfaces.py -scrapy/xlib/tx/endpoints.py -scrapy/xlib/tx/client.py -scrapy/xlib/tx/_newclient.py -scrapy/xlib/tx/__init__.py -scrapy/core/downloader/handlers/ftp.py scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c1683fb3e..e49a514b8 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -687,9 +687,6 @@ class BaseFTPTestCase(unittest.TestCase): password = "passwd" req_meta = {"ftp_user": username, "ftp_password": password} - if six.PY3: - skip = "Twisted missing ftp support for PY3" - def setUp(self): from twisted.protocols.ftp import FTPRealm, FTPFactory from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler @@ -700,8 +697,8 @@ class BaseFTPTestCase(unittest.TestCase): userdir = os.path.join(self.directory, self.username) os.mkdir(userdir) fp = FilePath(userdir) - fp.child('file.txt').setContent("I have the power!") - fp.child('file with spaces.txt').setContent("Moooooooooo power!") + fp.child('file.txt').setContent(b"I have the power!") + fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") # setup server realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) @@ -736,8 +733,8 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.status, 200) - self.assertEqual(r.body, 'I have the power!') - self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['17']}) + self.assertEqual(r.body, b'I have the power!') + self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -749,8 +746,8 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.status, 200) - self.assertEqual(r.body, 'Moooooooooo power!') - self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['18']}) + self.assertEqual(r.body, b'Moooooooooo power!') + self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']}) return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): @@ -763,7 +760,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): - local_fname = "/tmp/file.txt" + local_fname = b"/tmp/file.txt" meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, @@ -772,10 +769,10 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.body, local_fname) - self.assertEqual(r.headers, {'Local Filename': ['/tmp/file.txt'], 'Size': ['17']}) + self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']}) self.assertTrue(os.path.exists(local_fname)) - with open(local_fname) as f: - self.assertEqual(f.read(), "I have the power!") + with open(local_fname, "rb") as f: + self.assertEqual(f.read(), b"I have the power!") os.remove(local_fname) return self._add_test_callbacks(d, _test) @@ -810,8 +807,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase): os.mkdir(self.directory) fp = FilePath(self.directory) - fp.child('file.txt').setContent("I have the power!") - fp.child('file with spaces.txt').setContent("Moooooooooo power!") + fp.child('file.txt').setContent(b"I have the power!") + fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") # setup server for anonymous access realm = FTPRealm(anonymousRoot=self.directory)