diff --git a/requirements-py3.txt b/requirements-py3.txt new file mode 100644 index 000000000..81669da39 --- /dev/null +++ b/requirements-py3.txt @@ -0,0 +1,6 @@ +Twisted >= 15.1.0 +lxml>=3.2.4 +pyOpenSSL>=0.13.1 +cssselect>=0.9 +queuelib>=1.1.1 +w3lib>=1.8.0 diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index a619c349a..35050c13d 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -18,10 +18,10 @@ def _iter_command_classes(module_name): # TODO: add `name` attribute to commands and and merge this function with # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): - for obj in vars(module).itervalues(): + for obj in vars(module).values(): if inspect.isclass(obj) and \ - issubclass(obj, ScrapyCommand) and \ - obj.__module__ == module.__name__: + issubclass(obj, ScrapyCommand) and \ + obj.__module__ == module.__name__: yield obj def _get_commands_from_module(module, inproject): diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index f890300c4..311815b70 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,4 +1,4 @@ -from urlparse import unquote +from six.moves.urllib.parse import unquote from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index dcc588ef2..413a05dd1 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,7 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ - +import six from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred @@ -32,7 +32,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = method(request=request, spider=spider) assert response is None or isinstance(response, (Response, Request)), \ 'Middleware %s.process_request must return None, Response or Request, got %s' % \ - (method.im_self.__class__.__name__, response.__class__.__name__) + (six.get_method_self(method).__class__.__name__, response.__class__.__name__) if response: return response return download_func(request=request, spider=spider) @@ -46,7 +46,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = method(request=request, response=response, spider=spider) assert isinstance(response, (Response, Request)), \ 'Middleware %s.process_response must return Response or Request, got %s' % \ - (method.im_self.__class__.__name__, type(response)) + (six.get_method_self(method).__class__.__name__, type(response)) if isinstance(response, Request): return response return response @@ -57,7 +57,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = method(request=request, exception=exception, spider=spider) assert response is None or isinstance(response, (Response, Request)), \ 'Middleware %s.process_exception must return None, Response or Request, got %s' % \ - (method.im_self.__class__.__name__, type(response)) + (six.get_method_self(method).__class__.__name__, type(response)) if response: return response return _failure diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index f6bb62afb..c1c5b10fc 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,7 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ - +import six from twisted.python.failure import Failure from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred @@ -33,7 +33,9 @@ class SpiderMiddlewareManager(MiddlewareManager): self.methods['process_start_requests'].insert(0, mw.process_start_requests) def scrape_response(self, scrape_func, response, request, spider): - fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__) + fname = lambda f:'%s.%s' % ( + six.get_method_self(f).__class__.__name__, + six.get_method_function(f).__name__) def process_spider_input(response): for method in self.methods['process_spider_input']: diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index 14f4970b0..202340f53 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -3,7 +3,7 @@ HTMLParser-based link extractor """ import warnings -from HTMLParser import HTMLParser +from six.moves.html_parser import HTMLParser from six.moves.urllib.parse import urljoin from w3lib.url import safe_url_string diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 308d2f3c1..db49aff65 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -26,6 +26,7 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.misc import md5sum from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.python import to_bytes, to_native_str logger = logging.getLogger(__name__) @@ -198,7 +199,7 @@ class FilesPipeline(MediaPipeline): if age_days > self.EXPIRES: return # returning None force download - referer = request.headers.get('Referer') + referer = _get_referer(request) logger.debug( 'File (uptodate): Downloaded %(medianame)s from %(request)s ' 'referred in <%(referer)s>', @@ -224,7 +225,7 @@ class FilesPipeline(MediaPipeline): def media_failed(self, failure, request, info): if not isinstance(failure.value, IgnoreRequest): - referer = request.headers.get('Referer') + referer = _get_referer(request) logger.warning( 'File (unknown-error): Error downloading %(medianame)s from ' '%(request)s referred in <%(referer)s>: %(exception)s', @@ -236,7 +237,7 @@ class FilesPipeline(MediaPipeline): raise FileException def media_downloaded(self, response, request, info): - referer = request.headers.get('Referer') + referer = _get_referer(request) if response.status != 200: logger.warning( @@ -330,7 +331,7 @@ class FilesPipeline(MediaPipeline): return self.file_key(url) ## end of deprecation warning block - media_guid = hashlib.sha1(url).hexdigest() # change to request.url after deprecation + media_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation media_ext = os.path.splitext(url)[1] # change to request.url after deprecation return 'full/%s%s' % (media_guid, media_ext) @@ -338,3 +339,11 @@ class FilesPipeline(MediaPipeline): def file_key(self, url): return self.file_path(url) file_key._base = True + + +def _get_referer(request): + """ Return Referer HTTP header suitable for logging """ + referrer = request.headers.get('Referer') + if referrer is None: + return referrer + return to_native_str(referrer, errors='replace') diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 8b3bc2222..ff73b44b7 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -15,6 +15,7 @@ except ImportError: from PIL import Image from scrapy.utils.misc import md5sum +from scrapy.utils.python import to_bytes from scrapy.http import Request from scrapy.exceptions import DropItem #TODO: from scrapy.pipelines.media import MediaPipeline @@ -138,7 +139,7 @@ class ImagesPipeline(FilesPipeline): return self.image_key(url) ## end of deprecation warning block - image_guid = hashlib.sha1(url).hexdigest() # change to request.url after deprecation + image_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation return 'full/%s.jpg' % (image_guid) def thumb_path(self, request, thumb_id, response=None, info=None): @@ -163,7 +164,7 @@ class ImagesPipeline(FilesPipeline): return self.thumb_key(url, thumb_id) ## end of deprecation warning block - thumb_guid = hashlib.sha1(url).hexdigest() # change to request.url after deprecation + thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid) # deprecated diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index adddad093..f268e91ff 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -35,8 +35,8 @@ class TestProcessProtocol(protocol.ProcessProtocol): def __init__(self): self.deferred = defer.Deferred() - self.out = '' - self.err = '' + self.out = b'' + self.err = b'' self.exitcode = None def outReceived(self, data): diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 038f715a6..9be3a99a8 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,12 +1,9 @@ tests/test_closespider.py -tests/test_cmdline/__init__.py tests/test_command_fetch.py tests/test_command_shell.py tests/test_commands.py -tests/test_command_version.py tests/test_exporters.py tests/test_linkextractors.py -tests/test_loader.py tests/test_crawl.py tests/test_crawler.py tests/test_downloader_handlers.py diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt new file mode 100644 index 000000000..8f9e22f0b --- /dev/null +++ b/tests/requirements-py3.txt @@ -0,0 +1,4 @@ +pytest>=2.6.0 +pytest-twisted +testfixtures +jmespath diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 00fce2fbc..1e2905e95 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,9 +1,18 @@ +import os import sys +import shutil +import pstats +import tempfile from subprocess import Popen, PIPE import unittest +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO from scrapy.utils.test import get_testenv + class CmdlineTest(unittest.TestCase): def setUp(self): @@ -11,10 +20,11 @@ class CmdlineTest(unittest.TestCase): self.env['SCRAPY_SETTINGS_MODULE'] = 'tests.test_cmdline.settings' def _execute(self, *new_args, **kwargs): + encoding = getattr(sys.stdout, 'encoding') or 'utf-8' args = (sys.executable, '-m', 'scrapy.cmdline') + new_args proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) - comm = proc.communicate() - return comm[0].strip() + comm = proc.communicate()[0].strip() + return comm.decode(encoding) def test_default_settings(self): self.assertEqual(self._execute('settings', '--get', 'TEST1'), \ @@ -29,3 +39,18 @@ class CmdlineTest(unittest.TestCase): self.assertEqual(self._execute('settings', '--get', 'TEST1'), \ 'override') + def test_profiling(self): + path = tempfile.mkdtemp() + filename = os.path.join(path, 'res.prof') + try: + self._execute('version', '--profile', filename) + self.assertTrue(os.path.exists(filename)) + out = StringIO() + stats = pstats.Stats(filename, stream=out) + stats.print_stats() + out.seek(0) + stats = out.read() + self.assertIn('scrapy/commands/version.py', stats) + self.assertIn('tottime', stats) + finally: + shutil.rmtree(path) diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 6f0380d77..420713d87 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -1,3 +1,4 @@ +import sys from twisted.trial import unittest from twisted.internet import defer @@ -11,5 +12,6 @@ class VersionTest(ProcessTest, unittest.TestCase): @defer.inlineCallbacks def test_output(self): + encoding = getattr(sys.stdout, 'encoding') or 'utf-8' _, out, _ = yield self.execute([]) - self.assertEqual(out.strip(), "Scrapy %s" % scrapy.__version__) + self.assertEqual(out.strip().decode(encoding), "Scrapy %s" % scrapy.__version__) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f2ebf9c69..6d21acab0 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -141,7 +141,7 @@ class CrawlTestCase(TestCase): def test_unbounded_response(self): # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" - from urllib import urlencode + from six.moves.urllib.parse import urlencode query = urlencode({'raw': '''\ HTTP/1.1 200 OK Server: Apache-Coyote/1.1 diff --git a/tests/test_loader.py b/tests/test_loader.py index 6e8f7c0de..8cf5e484a 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,4 +1,5 @@ import unittest +import six from functools import partial from scrapy.loader import ItemLoader @@ -141,7 +142,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_get_value(self): il = NameItemLoader() - self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), unicode.upper)) + self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper)) self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) @@ -242,7 +243,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_extend_custom_input_processors(self): class ChildItemLoader(TestItemLoader): - name_in = MapCompose(TestItemLoader.name_in, unicode.swapcase) + name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase) il = ChildItemLoader() il.add_value('name', u'marta') @@ -250,7 +251,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_extend_default_input_processors(self): class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose(DefaultedItemLoader.default_input_processor, unicode.swapcase) + name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase) il = ChildDefaultedItemLoader() il.add_value('name', u'marta') @@ -423,7 +424,7 @@ class ProcessorsTest(unittest.TestCase): self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) self.assertEqual(proc(['', 'hello', 'world']), u' hello world') self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assert_(isinstance(proc(['hello', 'world']), unicode)) + self.assert_(isinstance(proc(['hello', 'world']), six.text_type)) def test_compose(self): proc = Compose(lambda v: v[0], str.upper) @@ -435,13 +436,13 @@ class ProcessorsTest(unittest.TestCase): def test_mapcompose(self): filter_world = lambda x: None if x == 'world' else x - proc = MapCompose(filter_world, unicode.upper) + proc = MapCompose(filter_world, six.text_type.upper) self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), [u'HELLO', u'THIS', u'IS', u'SCRAPY']) class SelectortemLoaderTest(unittest.TestCase): - response = HtmlResponse(url="", body=""" + response = HtmlResponse(url="", encoding='utf-8', body=b"""
marta
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index ac0438eba..c9977f5ca 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -12,6 +12,7 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings +from scrapy.utils.python import to_bytes from tests import mock @@ -103,7 +104,7 @@ class FilesPipelineTestCase(unittest.TestCase): class DeprecatedFilesPipeline(FilesPipeline): def file_key(self, url): - media_guid = hashlib.sha1(url).hexdigest() + media_guid = hashlib.sha1(to_bytes(url)).hexdigest() media_ext = os.path.splitext(url)[1] return 'empty/%s%s' % (media_guid, media_ext) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 04cec4b8e..f52fb4d3d 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -10,6 +10,7 @@ from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.pipelines.images import ImagesPipeline +from scrapy.utils.python import to_bytes skip = False try: @@ -100,11 +101,11 @@ class DeprecatedImagesPipeline(ImagesPipeline): return self.image_key(url) def image_key(self, url): - image_guid = hashlib.sha1(url).hexdigest() + image_guid = hashlib.sha1(to_bytes(url)).hexdigest() return 'empty/%s.jpg' % (image_guid) def thumb_key(self, url, thumb_id): - thumb_guid = hashlib.sha1(url).hexdigest() + thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid) diff --git a/tests/test_selector_csstranslator.py b/tests/test_selector_csstranslator.py index 7ef9003aa..1bc8882f8 100644 --- a/tests/test_selector_csstranslator.py +++ b/tests/test_selector_csstranslator.py @@ -9,7 +9,7 @@ from cssselect.parser import SelectorSyntaxError from cssselect.xpath import ExpressionError -HTMLBODY = ''' +HTMLBODY = b'''
diff --git a/tests/test_selector_lxmldocument.py b/tests/test_selector_lxmldocument.py index 7dab1d4b1..090cc21bc 100644 --- a/tests/test_selector_lxmldocument.py +++ b/tests/test_selector_lxmldocument.py @@ -6,7 +6,7 @@ from scrapy.http import TextResponse, HtmlResponse class LxmlDocumentTest(unittest.TestCase): def test_caching(self): - r1 = HtmlResponse('http://www.example.com', body='') + r1 = HtmlResponse('http://www.example.com', body=b'') r2 = r1.copy() doc1 = LxmlDocument(r1) @@ -19,7 +19,7 @@ class LxmlDocumentTest(unittest.TestCase): def test_null_char(self): # make sure bodies with null char ('\x00') don't raise a TypeError exception - body = 'test problematic \x00 body' + body = b'test problematic \x00 body' response = TextResponse('http://example.com/catalog/product/blabla-123', headers={'Content-Type': 'text/plain; charset=utf-8'}, body=body) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index a7042a6cf..f2780dcf1 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -13,7 +13,7 @@ class XmliterTestCase(unittest.TestCase): xmliter = staticmethod(xmliter) def test_xmliter(self): - body = """\ + body = b"""\ \ \ Type 1\ @@ -40,7 +40,7 @@ class XmliterTestCase(unittest.TestCase): [[u'one'], [u'two']]) def test_xmliter_namespaces(self): - body = """\ + body = b"""\ @@ -83,7 +83,7 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(StopIteration, next, iter) def test_xmliter_encoding(self): - body = '\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' + body = b'\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' response = XmlResponse('http://www.example.com', body=body) self.assertEqual( self.xmliter(response, 'item').next().extract(), @@ -95,7 +95,7 @@ class LxmlXmliterTestCase(XmliterTestCase): xmliter = staticmethod(xmliter_lxml) def test_xmliter_iterate_namespace(self): - body = """\ + body = b"""\ @@ -124,7 +124,7 @@ class LxmlXmliterTestCase(XmliterTestCase): self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg']) def test_xmliter_namespaces_prefix(self): - body = """\ + body = b"""\ diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 40c44f7d9..a62f13e21 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -20,7 +20,7 @@ class RequestSerializationTest(unittest.TestCase): callback='parse_item', errback='handle_error', method="POST", - body="some body", + body=b"some body", headers={'content-encoding': 'text/html; charset=latin-1'}, cookies={'currency': u'руб'}, encoding='latin-1', diff --git a/tox.ini b/tox.ini index 5c8c8c78d..fe6b5de38 100644 --- a/tox.ini +++ b/tox.ini @@ -40,18 +40,11 @@ commands = [testenv:py33] basepython = python3.3 deps = - Twisted >= 15.1.0 - lxml>=3.2.4 - pyOpenSSL>=0.13.1 - cssselect>=0.9 - queuelib>=1.1.1 - w3lib>=1.8.0 + -rrequirements-py3.txt + # Extras Pillow service_identity - # tests requirements - pytest>=2.6.0 - pytest-twisted - testfixtures + -rtests/requirements-py3.txt [testenv:py34] basepython = python3.4