From 20b839b44ba5795aa0a6cf96ac71d4524072fcab Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 12:42:45 +0300 Subject: [PATCH 1/6] py3: pass first crawl test (test_follow_all): fix mock server --- tests/mockserver.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index e7953c4d4..336633f4b 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -7,6 +7,7 @@ from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource from twisted.internet import reactor, defer, ssl from scrapy import twisted_version +from scrapy.utils.python import to_bytes if twisted_version < (11, 0, 0): @@ -55,12 +56,12 @@ class LeafResource(Resource): class Follow(LeafResource): def render(self, request): - total = getarg(request, "total", 100, type=int) - show = getarg(request, "show", 1, type=int) - order = getarg(request, "order", "desc") - maxlatency = getarg(request, "maxlatency", 0, type=float) - n = getarg(request, "n", total, type=int) - if order == "rand": + total = getarg(request, b"total", 100, type=int) + show = getarg(request, b"show", 1, type=int) + order = getarg(request, b"order", b"desc") + maxlatency = getarg(request, b"maxlatency", 0, type=float) + n = getarg(request, b"n", total, type=int) + if order == b"rand": nlist = [random.randint(1, total) for _ in range(show)] else: # order == "desc" nlist = range(n, max(n - show, 0), -1) @@ -73,19 +74,19 @@ class Follow(LeafResource): s = """ """ args = request.args.copy() for nl in nlist: - args["n"] = [str(nl)] + args[b"n"] = [to_bytes(str(nl))] argstr = urlencode(args, doseq=True) s += "follow %d
" % (argstr, nl) s += """""" - request.write(s) + request.write(to_bytes(s)) request.finish() class Delay(LeafResource): def render_GET(self, request): - n = getarg(request, "n", 1, type=float) - b = getarg(request, "b", 1, type=int) + n = getarg(request, b"n", 1, type=float) + b = getarg(request, b"b", 1, type=int) if b: # send headers now and delay body request.write('') @@ -93,16 +94,16 @@ class Delay(LeafResource): return NOT_DONE_YET def _delayedRender(self, request, n): - request.write("Response delayed for %0.3f seconds\n" % n) + request.write(to_bytes("Response delayed for %0.3f seconds\n" % n)) request.finish() class Status(LeafResource): def render_GET(self, request): - n = getarg(request, "n", 200, type=int) + n = getarg(request, b"n", 200, type=int) request.setResponseCode(n) - return "" + return b"" class Raw(LeafResource): @@ -114,7 +115,7 @@ class Raw(LeafResource): render_POST = render_GET def _delayedRender(self, request): - raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n') + raw = getarg(request, b'raw', b'HTTP 1.1 200 OK\n') request.startedWriting = 1 request.write(raw) request.channel.transport.loseConnection() @@ -128,7 +129,7 @@ class Echo(LeafResource): 'headers': dict(request.requestHeaders.getAllRawHeaders()), 'body': request.content.read(), } - return json.dumps(output) + return to_bytes(json.dumps(output)) class Partial(LeafResource): @@ -146,7 +147,7 @@ class Partial(LeafResource): class Drop(Partial): def _delayedRender(self, request): - abort = getarg(request, "abort", 0, type=int) + abort = getarg(request, b"abort", 0, type=int) request.write(b"this connection will be dropped\n") tr = request.channel.transport try: From 0680950b9898aa77e4c494ab8a318d791ef0d55f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 13:06:31 +0300 Subject: [PATCH 2/6] py3: pass CrawlTestCase.test_referer_header, fixing Echo resource in mockserver and json decoding in test --- tests/mockserver.py | 9 +++++---- tests/test_crawl.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 336633f4b..6877c786c 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,13 +1,12 @@ from __future__ import print_function import sys, time, random, os, json -import six from six.moves.urllib.parse import urlencode from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource from twisted.internet import reactor, defer, ssl from scrapy import twisted_version -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, to_unicode if twisted_version < (11, 0, 0): @@ -126,8 +125,10 @@ class Echo(LeafResource): def render_GET(self, request): output = { - 'headers': dict(request.requestHeaders.getAllRawHeaders()), - 'body': request.content.read(), + 'headers': dict( + (to_unicode(k), [to_unicode(v) for v in vs]) + for k, vs in request.requestHeaders.getAllRawHeaders()), + 'body': to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 814eb30d2..90fd921c8 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -8,6 +8,7 @@ from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.crawler import CrawlerRunner +from scrapy.utils.python import to_unicode from tests import mock from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider @@ -201,16 +202,16 @@ with multiples lines self.assertIn('responses', crawler.spider.meta) self.assertNotIn('failures', crawler.spider.meta) # start requests doesn't set Referer header - echo0 = json.loads(crawler.spider.meta['responses'][2].body) + echo0 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body)) self.assertNotIn('Referer', echo0['headers']) # following request sets Referer to start request url - echo1 = json.loads(crawler.spider.meta['responses'][1].body) + echo1 = json.loads(to_unicode(crawler.spider.meta['responses'][1].body)) self.assertEqual(echo1['headers'].get('Referer'), [req0.url]) # next request avoids Referer header - echo2 = json.loads(crawler.spider.meta['responses'][2].body) + echo2 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body)) self.assertNotIn('Referer', echo2['headers']) # last request explicitly sets a Referer header - echo3 = json.loads(crawler.spider.meta['responses'][3].body) + echo3 = json.loads(to_unicode(crawler.spider.meta['responses'][3].body)) self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com']) @defer.inlineCallbacks From ad2b3321b90245e59b2e4b99eb7a4296f6c7e768 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 13:13:41 +0300 Subject: [PATCH 3/6] py3 compat: use range, fixes CrawlTestCase.test_start_requests_bug_yielding --- tests/spiders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/spiders.py b/tests/spiders.py index 516062929..711d80cac 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -119,7 +119,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): if self.fail_before_yield: 1 / 0 - for s in xrange(100): + for s in range(100): qargs = {'total': 10, 'seed': s} url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) yield Request(url, meta={'seed': s}) From bf5f54fa339b44fec3451c88a78e4620f56c3bc8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:01:30 +0300 Subject: [PATCH 4/6] py3: fix getarg --- tests/mockserver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 6877c786c..365ec81fd 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -30,9 +30,12 @@ else: from twisted.internet.task import deferLater -def getarg(request, name, default=None, type=str): +def getarg(request, name, default=None, type=None): if name in request.args: - return type(request.args[name][0]) + value = request.args[name][0] + if type is not None: + value = type(value) + return value else: return default From 4607f2843e25b559f7483186916a1d91329c9dd8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 16:01:43 +0300 Subject: [PATCH 5/6] py3: unskip test_crawl --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 185a278fb..c8beea8a3 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,7 +1,6 @@ tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py -tests/test_crawl.py tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py From 5813de883888a69cf015ad506d6052e6191feb6e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 19 Jan 2016 18:08:05 +0300 Subject: [PATCH 6/6] py3: unskip test_closespider - it passes after fixing mockserver.Follow resouce on py3 --- tests/py3-ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index c8beea8a3..f189a4c86 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,4 +1,3 @@ -tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_mail.py