RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

This commit is contained in:
Taito Horiuchi 2017-02-28 14:34:06 +02:00
parent af23c920bf 0e5ed21397
commit b6f6d621b1
10 changed files with 57 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/core/downloader/handlers/ftp.py
View File

@ -39,12 +39,13 @@ from twisted.internet.protocol import Protocol, ClientCreator
from scrapy.http import Response
from scrapy.responsetypes import responsetypes
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
class ReceivedDataProtocol(Protocol):
def __init__(self, filename=None):
self.__filename = filename
self.body = open(filename, "w") if filename else BytesIO()
self.body = open(filename, "wb") if filename else BytesIO()
self.size = 0
def dataReceived(self, data):
@ -97,7 +98,7 @@ class FTPDownloadHandler(object):
protocol.close()
body = protocol.filename or protocol.body.read()
headers = {"local filename": protocol.filename or '', "size": protocol.size}
return respcls(url=request.url, status=200, body=body, headers=headers)
return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
def _failed(self, result, request):
message = result.getErrorMessage()
@ -106,6 +107,6 @@ class FTPDownloadHandler(object):
if m:
ftpcode = m.group()
httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
return Response(url=request.url, status=httpcode, body=message)
return Response(url=request.url, status=httpcode, body=to_bytes(message))
raise result.type(result.value)

9
scrapy/downloadermiddlewares/retry.py
View File

@ -22,6 +22,7 @@ from twisted.web.client import ResponseFailed
from scrapy.exceptions import NotConfigured
from scrapy.utils.response import response_status_message
from scrapy.core.downloader.handlers.http11 import TunnelError
from scrapy.utils.python import global_object_name
logger = logging.getLogger(__name__)
@ -62,6 +63,7 @@ class RetryMiddleware(object):
def _retry(self, request, reason, spider):
retries = request.meta.get('retry_times', 0) + 1
stats = spider.crawler.stats
if retries <= self.max_retry_times:
logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
{'request': request, 'retries': retries, 'reason': reason},
@ -70,8 +72,15 @@ class RetryMiddleware(object):
retryreq.meta['retry_times'] = retries
retryreq.dont_filter = True
retryreq.priority = request.priority + self.priority_adjust
if isinstance(reason, Exception):
reason = global_object_name(reason.__class__)
stats.inc_value('retry/count')
stats.inc_value('retry/reason_count/%s' % reason)
return retryreq
else:
stats.inc_value('retry/max_reached')
logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
{'request': request, 'retries': retries, 'reason': reason},
extra={'spider': spider})

4
scrapy/downloadermiddlewares/stats.py
View File

@ -1,6 +1,8 @@
from scrapy.exceptions import NotConfigured
from scrapy.utils.request import request_httprepr
from scrapy.utils.response import response_httprepr
from scrapy.utils.python import global_object_name
class DownloaderStats(object):
@ -27,6 +29,6 @@ class DownloaderStats(object):
return response
def process_exception(self, request, exception, spider):
ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__)
ex_class = global_object_name(exception.__class__)
self.stats.inc_value('downloader/exception_count', spider=spider)
self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)

2
scrapy/utils/misc.py
View File

@ -113,7 +113,7 @@ def md5sum(file):
m.update(d)
return m.hexdigest()
def rel_has_nofollow(rel):
"""Return True if link rel attribute has nofollow type"""
return True if rel is not None and 'nofollow' in rel.split() else False

11
scrapy/utils/python.py
View File

@ -344,3 +344,14 @@ def without_none_values(iterable):
return {k: v for k, v in six.iteritems(iterable) if v is not None}
except AttributeError:
return type(iterable)((v for v in iterable if v is not None))
def global_object_name(obj):
"""
Return full name of a global object.
>>> from scrapy import Request
>>> global_object_name(Request)
'scrapy.http.request.Request'
"""
return "%s.%s" % (obj.__module__, obj.__name__)

3
scrapy/utils/response.py
View File

@ -43,7 +43,8 @@ def get_meta_refresh(response):
def response_status_message(status):
"""Return status code plus status text descriptive message
"""
return '%s %s' % (status, to_native_str(http.RESPONSES.get(int(status), "Unknown Status")))
message = http.RESPONSES.get(int(status), "Unknown Status")
return '%s %s' % (status, to_native_str(message))
def response_httprepr(response):

7
tests/py3-ignores.txt
View File

@ -1,13 +1,6 @@
tests/test_linkextractors_deprecated.py
tests/test_proxy_connect.py
scrapy/xlib/tx/iweb.py
scrapy/xlib/tx/interfaces.py
scrapy/xlib/tx/endpoints.py
scrapy/xlib/tx/client.py
scrapy/xlib/tx/_newclient.py
scrapy/xlib/tx/__init__.py
scrapy/core/downloader/handlers/ftp.py
scrapy/linkextractors/sgml.py
scrapy/linkextractors/regex.py
scrapy/linkextractors/htmlparser.py

27
tests/test_downloader_handlers.py
View File

@ -687,9 +687,6 @@ class BaseFTPTestCase(unittest.TestCase):
password = "passwd"
req_meta = {"ftp_user": username, "ftp_password": password}
if six.PY3:
skip = "Twisted missing ftp support for PY3"
def setUp(self):
from twisted.protocols.ftp import FTPRealm, FTPFactory
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
@ -700,8 +697,8 @@ class BaseFTPTestCase(unittest.TestCase):
userdir = os.path.join(self.directory, self.username)
os.mkdir(userdir)
fp = FilePath(userdir)
fp.child('file.txt').setContent("I have the power!")
fp.child('file with spaces.txt').setContent("Moooooooooo power!")
fp.child('file.txt').setContent(b"I have the power!")
fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
# setup server
realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
@ -736,8 +733,8 @@ class BaseFTPTestCase(unittest.TestCase):
def _test(r):
self.assertEqual(r.status, 200)
self.assertEqual(r.body, 'I have the power!')
self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['17']})
self.assertEqual(r.body, b'I have the power!')
self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']})
return self._add_test_callbacks(d, _test)
def test_ftp_download_path_with_spaces(self):
@ -749,8 +746,8 @@ class BaseFTPTestCase(unittest.TestCase):
def _test(r):
self.assertEqual(r.status, 200)
self.assertEqual(r.body, 'Moooooooooo power!')
self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['18']})
self.assertEqual(r.body, b'Moooooooooo power!')
self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']})
return self._add_test_callbacks(d, _test)
def test_ftp_download_notexist(self):
@ -763,7 +760,7 @@ class BaseFTPTestCase(unittest.TestCase):
return self._add_test_callbacks(d, _test)
def test_ftp_local_filename(self):
local_fname = "/tmp/file.txt"
local_fname = b"/tmp/file.txt"
meta = {"ftp_local_filename": local_fname}
meta.update(self.req_meta)
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
@ -772,10 +769,10 @@ class BaseFTPTestCase(unittest.TestCase):
def _test(r):
self.assertEqual(r.body, local_fname)
self.assertEqual(r.headers, {'Local Filename': ['/tmp/file.txt'], 'Size': ['17']})
self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']})
self.assertTrue(os.path.exists(local_fname))
with open(local_fname) as f:
self.assertEqual(f.read(), "I have the power!")
with open(local_fname, "rb") as f:
self.assertEqual(f.read(), b"I have the power!")
os.remove(local_fname)
return self._add_test_callbacks(d, _test)
@ -810,8 +807,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
os.mkdir(self.directory)
fp = FilePath(self.directory)
fp.child('file.txt').setContent("I have the power!")
fp.child('file with spaces.txt').setContent("Moooooooooo power!")
fp.child('file.txt').setContent(b"I have the power!")
fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
# setup server for anonymous access
realm = FTPRealm(anonymousRoot=self.directory)

15
tests/test_downloadermiddleware_retry.py
View File

@ -13,9 +13,9 @@ from scrapy.utils.test import get_crawler
class RetryTest(unittest.TestCase):
def setUp(self):
crawler = get_crawler(Spider)
self.spider = crawler._create_spider('foo')
self.mw = RetryMiddleware.from_crawler(crawler)
self.crawler = get_crawler(Spider)
self.spider = self.crawler._create_spider('foo')
self.mw = RetryMiddleware.from_crawler(self.crawler)
self.mw.max_retry_times = 2
def test_priority_adjust(self):
@ -70,6 +70,10 @@ class RetryTest(unittest.TestCase):
# discard it
assert self.mw.process_response(req, rsp, self.spider) is rsp
assert self.crawler.stats.get_value('retry/max_reached') == 1
assert self.crawler.stats.get_value('retry/reason_count/503 Service Unavailable') == 2
assert self.crawler.stats.get_value('retry/count') == 2
def test_twistederrors(self):
exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,
DNSLookupError, ConnectionRefusedError, ConnectionDone,
@ -79,6 +83,11 @@ class RetryTest(unittest.TestCase):
req = Request('http://www.scrapytest.org/%s' % exc.__name__)
self._test_retry_exception(req, exc('foo'))
stats = self.crawler.stats
assert stats.get_value('retry/max_reached') == len(exceptions)
assert stats.get_value('retry/count') == len(exceptions) * 2
assert stats.get_value('retry/reason_count/twisted.internet.defer.TimeoutError') == 2
def _test_retry_exception(self, req, exception):
# first retry
req = self.mw.process_exception(req, exception, self.spider)

4
tests/test_proxy_connect.py
View File

@ -101,7 +101,9 @@ class ProxyConnectTestCase(TestCase):
self._assert_got_response_code(407, l)
def _assert_got_response_code(self, code, log):
print(log)
self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
def _assert_got_tunnel_error(self, log):
self.assertEqual(str(log).count('TunnelError'), 1)
print(log)
self.assertIn('TunnelError', str(log))