diff --git a/tests/mockserver.py b/tests/mockserver.py
index e7953c4d4..365ec81fd 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,12 +1,12 @@
from __future__ import print_function
import sys, time, random, os, json
-import six
from six.moves.urllib.parse import urlencode
from subprocess import Popen, PIPE
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.internet import reactor, defer, ssl
from scrapy import twisted_version
+from scrapy.utils.python import to_bytes, to_unicode
if twisted_version < (11, 0, 0):
@@ -30,9 +30,12 @@ else:
from twisted.internet.task import deferLater
-def getarg(request, name, default=None, type=str):
+def getarg(request, name, default=None, type=None):
if name in request.args:
- return type(request.args[name][0])
+ value = request.args[name][0]
+ if type is not None:
+ value = type(value)
+ return value
else:
return default
@@ -55,12 +58,12 @@ class LeafResource(Resource):
class Follow(LeafResource):
def render(self, request):
- total = getarg(request, "total", 100, type=int)
- show = getarg(request, "show", 1, type=int)
- order = getarg(request, "order", "desc")
- maxlatency = getarg(request, "maxlatency", 0, type=float)
- n = getarg(request, "n", total, type=int)
- if order == "rand":
+ total = getarg(request, b"total", 100, type=int)
+ show = getarg(request, b"show", 1, type=int)
+ order = getarg(request, b"order", b"desc")
+ maxlatency = getarg(request, b"maxlatency", 0, type=float)
+ n = getarg(request, b"n", total, type=int)
+ if order == b"rand":
nlist = [random.randint(1, total) for _ in range(show)]
else: # order == "desc"
nlist = range(n, max(n - show, 0), -1)
@@ -73,19 +76,19 @@ class Follow(LeafResource):
s = """ """
args = request.args.copy()
for nl in nlist:
- args["n"] = [str(nl)]
+ args[b"n"] = [to_bytes(str(nl))]
argstr = urlencode(args, doseq=True)
s += "follow %d
" % (argstr, nl)
s += """"""
- request.write(s)
+ request.write(to_bytes(s))
request.finish()
class Delay(LeafResource):
def render_GET(self, request):
- n = getarg(request, "n", 1, type=float)
- b = getarg(request, "b", 1, type=int)
+ n = getarg(request, b"n", 1, type=float)
+ b = getarg(request, b"b", 1, type=int)
if b:
# send headers now and delay body
request.write('')
@@ -93,16 +96,16 @@ class Delay(LeafResource):
return NOT_DONE_YET
def _delayedRender(self, request, n):
- request.write("Response delayed for %0.3f seconds\n" % n)
+ request.write(to_bytes("Response delayed for %0.3f seconds\n" % n))
request.finish()
class Status(LeafResource):
def render_GET(self, request):
- n = getarg(request, "n", 200, type=int)
+ n = getarg(request, b"n", 200, type=int)
request.setResponseCode(n)
- return ""
+ return b""
class Raw(LeafResource):
@@ -114,7 +117,7 @@ class Raw(LeafResource):
render_POST = render_GET
def _delayedRender(self, request):
- raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n')
+ raw = getarg(request, b'raw', b'HTTP 1.1 200 OK\n')
request.startedWriting = 1
request.write(raw)
request.channel.transport.loseConnection()
@@ -125,10 +128,12 @@ class Echo(LeafResource):
def render_GET(self, request):
output = {
- 'headers': dict(request.requestHeaders.getAllRawHeaders()),
- 'body': request.content.read(),
+ 'headers': dict(
+ (to_unicode(k), [to_unicode(v) for v in vs])
+ for k, vs in request.requestHeaders.getAllRawHeaders()),
+ 'body': to_unicode(request.content.read()),
}
- return json.dumps(output)
+ return to_bytes(json.dumps(output))
class Partial(LeafResource):
@@ -146,7 +151,7 @@ class Partial(LeafResource):
class Drop(Partial):
def _delayedRender(self, request):
- abort = getarg(request, "abort", 0, type=int)
+ abort = getarg(request, b"abort", 0, type=int)
request.write(b"this connection will be dropped\n")
tr = request.channel.transport
try:
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 185a278fb..f189a4c86 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,7 +1,5 @@
-tests/test_closespider.py
tests/test_exporters.py
tests/test_linkextractors_deprecated.py
-tests/test_crawl.py
tests/test_mail.py
tests/test_pipeline_files.py
tests/test_pipeline_images.py
diff --git a/tests/spiders.py b/tests/spiders.py
index 516062929..711d80cac 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -119,7 +119,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
if self.fail_before_yield:
1 / 0
- for s in xrange(100):
+ for s in range(100):
qargs = {'total': 10, 'seed': s}
url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1)
yield Request(url, meta={'seed': s})
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 814eb30d2..90fd921c8 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -8,6 +8,7 @@ from twisted.trial.unittest import TestCase
from scrapy.http import Request
from scrapy.crawler import CrawlerRunner
+from scrapy.utils.python import to_unicode
from tests import mock
from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
@@ -201,16 +202,16 @@ with multiples lines
self.assertIn('responses', crawler.spider.meta)
self.assertNotIn('failures', crawler.spider.meta)
# start requests doesn't set Referer header
- echo0 = json.loads(crawler.spider.meta['responses'][2].body)
+ echo0 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body))
self.assertNotIn('Referer', echo0['headers'])
# following request sets Referer to start request url
- echo1 = json.loads(crawler.spider.meta['responses'][1].body)
+ echo1 = json.loads(to_unicode(crawler.spider.meta['responses'][1].body))
self.assertEqual(echo1['headers'].get('Referer'), [req0.url])
# next request avoids Referer header
- echo2 = json.loads(crawler.spider.meta['responses'][2].body)
+ echo2 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body))
self.assertNotIn('Referer', echo2['headers'])
# last request explicitly sets a Referer header
- echo3 = json.loads(crawler.spider.meta['responses'][3].body)
+ echo3 = json.loads(to_unicode(crawler.spider.meta['responses'][3].body))
self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com'])
@defer.inlineCallbacks