From 3ffa2c5759d068e298d94be406fc26faffac6034 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 20 Oct 2016 18:01:50 +0200 Subject: [PATCH] Support Anonymous FTP Fixes GH-2342 --- scrapy/core/downloader/handlers/ftp.py | 20 ++++++---- scrapy/settings/default_settings.py | 3 ++ tests/test_downloader_handlers.py | 52 +++++++++++++++++++++++--- 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 93fa0a3a7..603c1e2e3 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys: import re from io import BytesIO -from six.moves.urllib.parse import urlparse, unquote +from six.moves.urllib.parse import unquote from twisted.internet import reactor from twisted.protocols.ftp import FTPClient, CommandFailed @@ -38,6 +38,8 @@ from twisted.internet.protocol import Protocol, ClientCreator from scrapy.http import Response from scrapy.responsetypes import responsetypes +from scrapy.utils.httpobj import urlparse_cached + class ReceivedDataProtocol(Protocol): def __init__(self, filename=None): @@ -64,14 +66,18 @@ class FTPDownloadHandler(object): "default": 503, } - def __init__(self, setting): - pass + def __init__(self, settings): + self.anonymous_user = settings['FTP_ANONYMOUS_USER'] + self.anonymous_password = settings['FTP_ANONYMOUS_PASSWORD'] def download_request(self, request, spider): - parsed_url = urlparse(request.url) - creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"], - request.meta["ftp_password"], - passive=request.meta.get("ftp_passive", 1)) + parsed_url = urlparse_cached(request) + user = request.meta.get("ftp_user", self.anonymous_user) + password = request.meta.get("ftp_password") + if user == self.anonymous_user and password is None: + password = self.anonymous_password + creator = ClientCreator(reactor, FTPClient, user, password, + passive=request.meta.get("ftp_passive", 1)) return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient, request, unquote(parsed_url.path)) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index cb88bc2bf..a21c2584a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -161,6 +161,9 @@ FEED_EXPORTERS_BASE = { FILES_STORE_S3_ACL = 'private' +FTP_ANONYMOUS_USER = 'anonymous' +FTP_ANONYMOUS_PASSWORD = 'anonymous@example.com' + HTTPCACHE_ENABLED = False HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 6333efceb..2d97573b3 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -676,10 +676,11 @@ class S3TestCase(unittest.TestCase): b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=') -class FTPTestCase(unittest.TestCase): +class BaseFTPTestCase(unittest.TestCase): username = "scrapy" password = "passwd" + req_meta = {"ftp_user": username, "ftp_password": password} if six.PY3: skip = "Twisted missing ftp support for PY3" @@ -722,7 +723,7 @@ class FTPTestCase(unittest.TestCase): def test_ftp_download_success(self): request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, - meta={"ftp_user": self.username, "ftp_password": self.password}) + meta=self.req_meta) d = self.download_handler.download_request(request, None) def _test(r): @@ -734,7 +735,7 @@ class FTPTestCase(unittest.TestCase): def test_ftp_download_path_with_spaces(self): request = Request( url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum, - meta={"ftp_user": self.username, "ftp_password": self.password} + meta=self.req_meta ) d = self.download_handler.download_request(request, None) @@ -746,7 +747,7 @@ class FTPTestCase(unittest.TestCase): def test_ftp_download_notexist(self): request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, - meta={"ftp_user": self.username, "ftp_password": self.password}) + meta=self.req_meta) d = self.download_handler.download_request(request, None) def _test(r): @@ -755,8 +756,10 @@ class FTPTestCase(unittest.TestCase): def test_ftp_local_filename(self): local_fname = "/tmp/file.txt" + meta = {"ftp_local_filename": local_fname} + meta.update(self.req_meta) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, - meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname}) + meta=meta) d = self.download_handler.download_request(request, None) def _test(r): @@ -768,13 +771,50 @@ class FTPTestCase(unittest.TestCase): os.remove(local_fname) return self._add_test_callbacks(d, _test) + +class FTPTestCase(BaseFTPTestCase): + def test_invalid_credentials(self): from twisted.protocols.ftp import ConnectionLost + meta = dict(self.req_meta) + meta.update({"ftp_password": 'invalid'}) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, - meta={"ftp_user": self.username, "ftp_password": 'invalid'}) + meta=meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.type, ConnectionLost) return self._add_test_callbacks(d, errback=_test) + + +class AnonymousFTPTestCase(BaseFTPTestCase): + + username = "anonymous" + req_meta = {} + + def setUp(self): + from twisted.protocols.ftp import FTPRealm, FTPFactory + from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler + + # setup dir and test file + self.directory = self.mktemp() + os.mkdir(self.directory) + + fp = FilePath(self.directory) + fp.child('file.txt').setContent("I have the power!") + fp.child('file with spaces.txt').setContent("Moooooooooo power!") + + # setup server for anonymous access + realm = FTPRealm(anonymousRoot=self.directory) + p = portal.Portal(realm) + p.registerChecker(checkers.AllowAnonymousAccess(), + credentials.IAnonymous) + + self.factory = FTPFactory(portal=p, + userAnonymous=self.username) + print("self.factory.allowAnonymous=%r" % self.factory.allowAnonymous) + self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") + self.portNum = self.port.getHost().port + self.download_handler = FTPDownloadHandler(Settings()) + self.addCleanup(self.port.stopListening)