diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c1f488f73..f616742c4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -686,6 +686,42 @@ The Feed Temp dir allows you to set a custom folder to save crawler temporary files before uploading with :ref:`FTP feed storage ` and :ref:`Amazon S3 `. +.. setting:: FTP_PASSIVE_MODE + +FTP_PASSIVE_MODE +---------------- + +Default: ``True`` + +Whether or not to use passive mode when initiating FTP transfers. + +.. setting:: FTP_PASSWORD + +FTP_PASSWORD +------------ + +Default: ``"guest"`` + +The password to use for FTP connections when there is no ``"ftp_password"`` +in ``Request`` meta. + +.. note:: + Paraphrasing `RFC 1635`_, although it is common to use either the password + "guest" or one's e-mail address for anonymous FTP, + some FTP servers explicitly ask for the user's e-mail address + and will not allow login with the "guest" password. + +.. _RFC 1635: https://tools.ietf.org/html/rfc1635 + +.. setting:: FTP_USER + +FTP_USER +-------- + +Default: ``"anonymous"`` + +The username to use for FTP connections when there is no ``"ftp_user"`` +in ``Request`` meta. .. setting:: ITEM_PIPELINES diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 93fa0a3a7..1398140b4 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys: import re from io import BytesIO -from six.moves.urllib.parse import urlparse, unquote +from six.moves.urllib.parse import unquote from twisted.internet import reactor from twisted.protocols.ftp import FTPClient, CommandFailed @@ -38,6 +38,8 @@ from twisted.internet.protocol import Protocol, ClientCreator from scrapy.http import Response from scrapy.responsetypes import responsetypes +from scrapy.utils.httpobj import urlparse_cached + class ReceivedDataProtocol(Protocol): def __init__(self, filename=None): @@ -64,14 +66,19 @@ class FTPDownloadHandler(object): "default": 503, } - def __init__(self, setting): - pass + def __init__(self, settings): + self.default_user = settings['FTP_USER'] + self.default_password = settings['FTP_PASSWORD'] + self.passive_mode = settings['FTP_PASSIVE_MODE'] def download_request(self, request, spider): - parsed_url = urlparse(request.url) - creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"], - request.meta["ftp_password"], - passive=request.meta.get("ftp_passive", 1)) + parsed_url = urlparse_cached(request) + user = request.meta.get("ftp_user", self.default_user) + password = request.meta.get("ftp_password", self.default_password) + passive_mode = 1 if bool(request.meta.get("ftp_passive", + self.passive_mode)) else 0 + creator = ClientCreator(reactor, FTPClient, user, password, + passive=passive_mode) return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient, request, unquote(parsed_url.path)) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2d01befbb..e0e39120c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -161,6 +161,10 @@ FEED_EXPORTERS_BASE = { FILES_STORE_S3_ACL = 'private' +FTP_USER = 'anonymous' +FTP_PASSWORD = 'guest' +FTP_PASSIVE_MODE = True + HTTPCACHE_ENABLED = False HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index f9980f7b2..c1683fb3e 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -6,6 +6,7 @@ try: from unittest import mock except ImportError: import mock +import shutil from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory @@ -680,10 +681,11 @@ class S3TestCase(unittest.TestCase): b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=') -class FTPTestCase(unittest.TestCase): +class BaseFTPTestCase(unittest.TestCase): username = "scrapy" password = "passwd" + req_meta = {"ftp_user": username, "ftp_password": password} if six.PY3: skip = "Twisted missing ftp support for PY3" @@ -729,7 +731,7 @@ class FTPTestCase(unittest.TestCase): def test_ftp_download_success(self): request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, - meta={"ftp_user": self.username, "ftp_password": self.password}) + meta=self.req_meta) d = self.download_handler.download_request(request, None) def _test(r): @@ -741,7 +743,7 @@ class FTPTestCase(unittest.TestCase): def test_ftp_download_path_with_spaces(self): request = Request( url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum, - meta={"ftp_user": self.username, "ftp_password": self.password} + meta=self.req_meta ) d = self.download_handler.download_request(request, None) @@ -753,7 +755,7 @@ class FTPTestCase(unittest.TestCase): def test_ftp_download_notexist(self): request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, - meta={"ftp_user": self.username, "ftp_password": self.password}) + meta=self.req_meta) d = self.download_handler.download_request(request, None) def _test(r): @@ -762,8 +764,10 @@ class FTPTestCase(unittest.TestCase): def test_ftp_local_filename(self): local_fname = "/tmp/file.txt" + meta = {"ftp_local_filename": local_fname} + meta.update(self.req_meta) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, - meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname}) + meta=meta) d = self.download_handler.download_request(request, None) def _test(r): @@ -775,13 +779,52 @@ class FTPTestCase(unittest.TestCase): os.remove(local_fname) return self._add_test_callbacks(d, _test) + +class FTPTestCase(BaseFTPTestCase): + def test_invalid_credentials(self): from twisted.protocols.ftp import ConnectionLost + meta = dict(self.req_meta) + meta.update({"ftp_password": 'invalid'}) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, - meta={"ftp_user": self.username, "ftp_password": 'invalid'}) + meta=meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.type, ConnectionLost) return self._add_test_callbacks(d, errback=_test) + + +class AnonymousFTPTestCase(BaseFTPTestCase): + + username = "anonymous" + req_meta = {} + + def setUp(self): + from twisted.protocols.ftp import FTPRealm, FTPFactory + from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler + + # setup dir and test file + self.directory = self.mktemp() + os.mkdir(self.directory) + + fp = FilePath(self.directory) + fp.child('file.txt').setContent("I have the power!") + fp.child('file with spaces.txt').setContent("Moooooooooo power!") + + # setup server for anonymous access + realm = FTPRealm(anonymousRoot=self.directory) + p = portal.Portal(realm) + p.registerChecker(checkers.AllowAnonymousAccess(), + credentials.IAnonymous) + + self.factory = FTPFactory(portal=p, + userAnonymous=self.username) + self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") + self.portNum = self.port.getHost().port + self.download_handler = FTPDownloadHandler(Settings()) + self.addCleanup(self.port.stopListening) + + def tearDown(self): + shutil.rmtree(self.directory)