mirror of https://github.com/scrapy/scrapy.git
Merge pull request #1386 from nyov/tmp-py3
[MRG+1] [py3] replace rfc822 with email.utils
This commit is contained in:
Mikhail Korobov
commit
dafcfd5be6
|
|
@ -7,9 +7,9 @@ See documentation in topics/media-pipeline.rst
|
|||
import hashlib
|
||||
import os
|
||||
import os.path
|
||||
import rfc822
|
||||
import time
|
||||
import logging
|
||||
from email.utils import parsedate_tz, mktime_tz
|
||||
from six.moves.urllib.parse import urlparse
|
||||
from collections import defaultdict
|
||||
import six
|
||||
|
|
@ -91,8 +91,8 @@ class S3FilesStore(object):
|
|||
def _onsuccess(boto_key):
|
||||
checksum = boto_key.etag.strip('"')
|
||||
last_modified = boto_key.last_modified
|
||||
modified_tuple = rfc822.parsedate_tz(last_modified)
|
||||
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
|
||||
modified_tuple = parsedate_tz(last_modified)
|
||||
modified_stamp = int(mktime_tz(modified_tuple))
|
||||
return {'checksum': checksum, 'last_modified': modified_stamp}
|
||||
|
||||
return self._get_boto_key(path).addCallback(_onsuccess)
|
||||
|
|
|
|||
|
|
@ -92,9 +92,9 @@ class ResponseTypes(object):
|
|||
chunk = body[:5000]
|
||||
if isbinarytext(chunk):
|
||||
return self.from_mimetype('application/octet-stream')
|
||||
elif "<html>" in chunk.lower():
|
||||
elif b"<html>" in chunk.lower():
|
||||
return self.from_mimetype('text/html')
|
||||
elif "<?xml" in chunk.lower():
|
||||
elif b"<?xml" in chunk.lower():
|
||||
return self.from_mimetype('text/xml')
|
||||
else:
|
||||
return self.from_mimetype('text')
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ def iter_errback(iterable, errback, *a, **kw):
|
|||
iterating it.
|
||||
"""
|
||||
it = iter(iterable)
|
||||
while 1:
|
||||
while True:
|
||||
try:
|
||||
yield next(it)
|
||||
except StopIteration:
|
||||
|
|
|
|||
|
|
@ -106,7 +106,7 @@ def md5sum(file):
|
|||
'784406af91dd5a54fbb9c84c2236595a'
|
||||
"""
|
||||
m = hashlib.md5()
|
||||
while 1:
|
||||
while True:
|
||||
d = file.read(8096)
|
||||
if not d:
|
||||
break
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ tests/test_crawler.py
|
|||
tests/test_downloader_handlers.py
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py
|
||||
tests/test_downloadermiddleware_cookies.py
|
||||
tests/test_downloadermiddleware_decompression.py
|
||||
tests/test_downloadermiddleware_defaultheaders.py
|
||||
tests/test_downloadermiddleware_downloadtimeout.py
|
||||
tests/test_downloadermiddleware_httpauth.py
|
||||
|
|
@ -28,10 +27,8 @@ tests/test_engine.py
|
|||
tests/test_http_cookies.py
|
||||
tests/test_logformatter.py
|
||||
tests/test_mail.py
|
||||
tests/test_middleware.py
|
||||
tests/test_pipeline_files.py
|
||||
tests/test_pipeline_images.py
|
||||
tests/test_pipeline_media.py
|
||||
tests/test_proxy_connect.py
|
||||
tests/test_responsetypes.py
|
||||
tests/test_selector_csstranslator.py
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class DecompressionMiddlewareTest(TestCase):
|
|||
assert_samelines(self, new.body, rsp.body)
|
||||
|
||||
def test_empty_response(self):
|
||||
rsp = Response(url='http://test.com', body='')
|
||||
rsp = Response(url='http://test.com', body=b'')
|
||||
new = self.mw.process_response(None, rsp, self.spider)
|
||||
assert new is rsp
|
||||
assert not rsp.body
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from twisted.trial import unittest
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.middleware import MiddlewareManager
|
||||
import six
|
||||
|
||||
class M1(object):
|
||||
|
||||
|
|
@ -65,12 +66,20 @@ class MiddlewareManagerTest(unittest.TestCase):
|
|||
|
||||
def test_methods(self):
|
||||
mwman = TestMiddlewareManager(M1(), M2(), M3())
|
||||
self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
|
||||
[M1, M2])
|
||||
self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
|
||||
[M2, M1])
|
||||
self.assertEqual([x.im_class for x in mwman.methods['process']],
|
||||
[M1, M3])
|
||||
if six.PY2:
|
||||
self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
|
||||
[M1, M2])
|
||||
self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
|
||||
[M2, M1])
|
||||
self.assertEqual([x.im_class for x in mwman.methods['process']],
|
||||
[M1, M3])
|
||||
else:
|
||||
self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
|
||||
[M1, M2])
|
||||
self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
|
||||
[M2, M1])
|
||||
self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
|
||||
[M1, M3])
|
||||
|
||||
def test_enabled(self):
|
||||
m1, m2, m3 = M1(), M2(), M3()
|
||||
|
|
|
|||
|
|
@ -192,7 +192,7 @@ def _create_item_with_files(*files):
|
|||
def _prepare_request_object(item_url):
|
||||
return Request(
|
||||
item_url,
|
||||
meta={'response': Response(item_url, status=200, body='data')})
|
||||
meta={'response': Response(item_url, status=200, body=b'data')})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
|
|||
|
||||
def test_default_media_downloaded(self):
|
||||
request = Request('http://url')
|
||||
response = Response('http://url', body='')
|
||||
response = Response('http://url', body=b'')
|
||||
assert self.pipe.media_downloaded(response, request, self.info) is response
|
||||
|
||||
def test_default_media_failed(self):
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ class ResponseTypesTest(unittest.TestCase):
|
|||
for source, cls in mappings:
|
||||
retcls = responsetypes.from_body(source)
|
||||
assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls)
|
||||
|
||||
|
||||
def test_from_headers(self):
|
||||
mappings = [
|
||||
({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse),
|
||||
|
|
|
|||
Loading…
Reference in New Issue