From 683ef2a8d9666bbf20462a35d01d517ad1eca73e Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 25 Jul 2015 16:44:28 +0000 Subject: [PATCH 1/4] replace rfc822 with email.utils --- scrapy/pipelines/files.py | 6 +++--- scrapy/utils/defer.py | 2 +- scrapy/utils/misc.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 250f46ad8..a449793c9 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -7,9 +7,9 @@ See documentation in topics/media-pipeline.rst import hashlib import os import os.path -import rfc822 import time import logging +from email.utils import parsedate_tz, mktime_tz from six.moves.urllib.parse import urlparse from collections import defaultdict import six @@ -91,8 +91,8 @@ class S3FilesStore(object): def _onsuccess(boto_key): checksum = boto_key.etag.strip('"') last_modified = boto_key.last_modified - modified_tuple = rfc822.parsedate_tz(last_modified) - modified_stamp = int(rfc822.mktime_tz(modified_tuple)) + modified_tuple = parsedate_tz(last_modified) + modified_stamp = int(mktime_tz(modified_tuple)) return {'checksum': checksum, 'last_modified': modified_stamp} return self._get_boto_key(path).addCallback(_onsuccess) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 7a3f20476..8f3824abf 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -97,7 +97,7 @@ def iter_errback(iterable, errback, *a, **kw): iterating it. """ it = iter(iterable) - while 1: + while True: try: yield next(it) except StopIteration: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index c269b7f74..4215e41d2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -106,7 +106,7 @@ def md5sum(file): '784406af91dd5a54fbb9c84c2236595a' """ m = hashlib.md5() - while 1: + while True: d = file.read(8096) if not d: break From 8ab1648a363090d8fc22547cb3b763d79ed30c7d Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 25 Jul 2015 16:58:30 +0000 Subject: [PATCH 2/4] PY3 fix test middleware --- tests/py3-ignores.txt | 1 - tests/test_middleware.py | 21 +++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 4dfab4a9a..a6c92d508 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -28,7 +28,6 @@ tests/test_engine.py tests/test_http_cookies.py tests/test_logformatter.py tests/test_mail.py -tests/test_middleware.py tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_pipeline_media.py diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 48131462c..b6d885330 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -3,6 +3,7 @@ from twisted.trial import unittest from scrapy.settings import Settings from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager +import six class M1(object): @@ -65,12 +66,20 @@ class MiddlewareManagerTest(unittest.TestCase): def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3()) - self.assertEqual([x.im_class for x in mwman.methods['open_spider']], - [M1, M2]) - self.assertEqual([x.im_class for x in mwman.methods['close_spider']], - [M2, M1]) - self.assertEqual([x.im_class for x in mwman.methods['process']], - [M1, M3]) + if six.PY2: + self.assertEqual([x.im_class for x in mwman.methods['open_spider']], + [M1, M2]) + self.assertEqual([x.im_class for x in mwman.methods['close_spider']], + [M2, M1]) + self.assertEqual([x.im_class for x in mwman.methods['process']], + [M1, M3]) + else: + self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']], + [M1, M2]) + self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']], + [M2, M1]) + self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']], + [M1, M3]) def test_enabled(self): m1, m2, m3 = M1(), M2(), M3() From ec8afbc060fdd4b5159470c3964f4482ccf4dbe3 Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 25 Jul 2015 17:56:46 +0000 Subject: [PATCH 3/4] PY3 fix test pipeline media --- tests/py3-ignores.txt | 1 - tests/test_pipeline_files.py | 2 +- tests/test_pipeline_media.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index a6c92d508..96b07ae28 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -30,7 +30,6 @@ tests/test_logformatter.py tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py -tests/test_pipeline_media.py tests/test_proxy_connect.py tests/test_responsetypes.py tests/test_selector_csstranslator.py diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index b12f41174..ac0438eba 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -192,7 +192,7 @@ def _create_item_with_files(*files): def _prepare_request_object(item_url): return Request( item_url, - meta={'response': Response(item_url, status=200, body='data')}) + meta={'response': Response(item_url, status=200, body=b'data')}) if __name__ == "__main__": diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index fd8b28ce1..f30b4fea3 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -44,7 +44,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): def test_default_media_downloaded(self): request = Request('http://url') - response = Response('http://url', body='') + response = Response('http://url', body=b'') assert self.pipe.media_downloaded(response, request, self.info) is response def test_default_media_failed(self): From e044bfa60f072aca25e5561136dc0b9e89b2a1ed Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 25 Jul 2015 20:51:27 +0000 Subject: [PATCH 4/4] PY3 fix test downloadermiddleware decompression --- scrapy/responsetypes.py | 4 ++-- tests/py3-ignores.txt | 1 - tests/test_downloadermiddleware_decompression.py | 2 +- tests/test_responsetypes.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 7c017feef..c212f5706 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -92,9 +92,9 @@ class ResponseTypes(object): chunk = body[:5000] if isbinarytext(chunk): return self.from_mimetype('application/octet-stream') - elif "" in chunk.lower(): + elif b"" in chunk.lower(): return self.from_mimetype('text/html') - elif " %s != %s" % (source, retcls, cls) - + def test_from_headers(self): mappings = [ ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse),