RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support Anonymous FTP 

Fixes GH-2342
This commit is contained in:
Paul Tremberth 2016-10-20 18:01:50 +02:00
parent afac3fd2c2
commit 3ffa2c5759
3 changed files with 62 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
scrapy/core/downloader/handlers/ftp.py
View File

@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
import re
from io import BytesIO
from six.moves.urllib.parse import urlparse, unquote
from six.moves.urllib.parse import unquote
from twisted.internet import reactor
from twisted.protocols.ftp import FTPClient, CommandFailed
@ -38,6 +38,8 @@ from twisted.internet.protocol import Protocol, ClientCreator
from scrapy.http import Response
from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
class ReceivedDataProtocol(Protocol):
def __init__(self, filename=None):
@ -64,14 +66,18 @@ class FTPDownloadHandler(object):
"default": 503,
}
def __init__(self, setting):
pass
def __init__(self, settings):
self.anonymous_user = settings['FTP_ANONYMOUS_USER']
self.anonymous_password = settings['FTP_ANONYMOUS_PASSWORD']
def download_request(self, request, spider):
parsed_url = urlparse(request.url)
creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"],
request.meta["ftp_password"],
passive=request.meta.get("ftp_passive", 1))
parsed_url = urlparse_cached(request)
user = request.meta.get("ftp_user", self.anonymous_user)
password = request.meta.get("ftp_password")
if user == self.anonymous_user and password is None:
password = self.anonymous_password
creator = ClientCreator(reactor, FTPClient, user, password,
passive=request.meta.get("ftp_passive", 1))
return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
request, unquote(parsed_url.path))

3
scrapy/settings/default_settings.py
View File

@ -161,6 +161,9 @@ FEED_EXPORTERS_BASE = {
FILES_STORE_S3_ACL = 'private'
FTP_ANONYMOUS_USER = 'anonymous'
FTP_ANONYMOUS_PASSWORD = 'anonymous@example.com'
HTTPCACHE_ENABLED = False
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_IGNORE_MISSING = False

52
tests/test_downloader_handlers.py
View File

@ -676,10 +676,11 @@ class S3TestCase(unittest.TestCase):
b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
class FTPTestCase(unittest.TestCase):
class BaseFTPTestCase(unittest.TestCase):
username = "scrapy"
password = "passwd"
req_meta = {"ftp_user": username, "ftp_password": password}
if six.PY3:
skip = "Twisted missing ftp support for PY3"
@ -722,7 +723,7 @@ class FTPTestCase(unittest.TestCase):
def test_ftp_download_success(self):
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
meta={"ftp_user": self.username, "ftp_password": self.password})
meta=self.req_meta)
d = self.download_handler.download_request(request, None)
def _test(r):
@ -734,7 +735,7 @@ class FTPTestCase(unittest.TestCase):
def test_ftp_download_path_with_spaces(self):
request = Request(
url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum,
meta={"ftp_user": self.username, "ftp_password": self.password}
meta=self.req_meta
)
d = self.download_handler.download_request(request, None)
@ -746,7 +747,7 @@ class FTPTestCase(unittest.TestCase):
def test_ftp_download_notexist(self):
request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum,
meta={"ftp_user": self.username, "ftp_password": self.password})
meta=self.req_meta)
d = self.download_handler.download_request(request, None)
def _test(r):
@ -755,8 +756,10 @@ class FTPTestCase(unittest.TestCase):
def test_ftp_local_filename(self):
local_fname = "/tmp/file.txt"
meta = {"ftp_local_filename": local_fname}
meta.update(self.req_meta)
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname})
meta=meta)
d = self.download_handler.download_request(request, None)
def _test(r):
@ -768,13 +771,50 @@ class FTPTestCase(unittest.TestCase):
os.remove(local_fname)
return self._add_test_callbacks(d, _test)
class FTPTestCase(BaseFTPTestCase):
def test_invalid_credentials(self):
from twisted.protocols.ftp import ConnectionLost
meta = dict(self.req_meta)
meta.update({"ftp_password": 'invalid'})
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
meta={"ftp_user": self.username, "ftp_password": 'invalid'})
meta=meta)
d = self.download_handler.download_request(request, None)
def _test(r):
self.assertEqual(r.type, ConnectionLost)
return self._add_test_callbacks(d, errback=_test)
class AnonymousFTPTestCase(BaseFTPTestCase):
username = "anonymous"
req_meta = {}
def setUp(self):
from twisted.protocols.ftp import FTPRealm, FTPFactory
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
# setup dir and test file
self.directory = self.mktemp()
os.mkdir(self.directory)
fp = FilePath(self.directory)
fp.child('file.txt').setContent("I have the power!")
fp.child('file with spaces.txt').setContent("Moooooooooo power!")
# setup server for anonymous access
realm = FTPRealm(anonymousRoot=self.directory)
p = portal.Portal(realm)
p.registerChecker(checkers.AllowAnonymousAccess(),
credentials.IAnonymous)
self.factory = FTPFactory(portal=p,
userAnonymous=self.username)
print("self.factory.allowAnonymous=%r" % self.factory.allowAnonymous)
self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
self.portNum = self.port.getHost().port
self.download_handler = FTPDownloadHandler(Settings())
self.addCleanup(self.port.stopListening)