RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1692 from lopuhin/py3-test-crawl 

[MRG+1] Py3: port test crawl
This commit is contained in:
Elias Dorneles 2016-01-21 13:05:20 -02:00
parent 0f500a1f82 5813de8838
commit e0074d7f4d
4 changed files with 32 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
tests/mockserver.py
View File

@ -1,12 +1,12 @@
from __future__ import print_function
import sys, time, random, os, json
import six
from six.moves.urllib.parse import urlencode
from subprocess import Popen, PIPE
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.internet import reactor, defer, ssl
from scrapy import twisted_version
from scrapy.utils.python import to_bytes, to_unicode
if twisted_version < (11, 0, 0):
@ -30,9 +30,12 @@ else:
from twisted.internet.task import deferLater
def getarg(request, name, default=None, type=str):
def getarg(request, name, default=None, type=None):
if name in request.args:
return type(request.args[name][0])
value = request.args[name][0]
if type is not None:
value = type(value)
return value
else:
return default
@ -55,12 +58,12 @@ class LeafResource(Resource):
class Follow(LeafResource):
def render(self, request):
total = getarg(request, "total", 100, type=int)
show = getarg(request, "show", 1, type=int)
order = getarg(request, "order", "desc")
maxlatency = getarg(request, "maxlatency", 0, type=float)
n = getarg(request, "n", total, type=int)
if order == "rand":
total = getarg(request, b"total", 100, type=int)
show = getarg(request, b"show", 1, type=int)
order = getarg(request, b"order", b"desc")
maxlatency = getarg(request, b"maxlatency", 0, type=float)
n = getarg(request, b"n", total, type=int)
if order == b"rand":
nlist = [random.randint(1, total) for _ in range(show)]
else: # order == "desc"
nlist = range(n, max(n - show, 0), -1)
@ -73,19 +76,19 @@ class Follow(LeafResource):
s = """<html> <head></head> <body>"""
args = request.args.copy()
for nl in nlist:
args["n"] = [str(nl)]
args[b"n"] = [to_bytes(str(nl))]
argstr = urlencode(args, doseq=True)
s += "<a href='/follow?%s'>follow %d</a><br>" % (argstr, nl)
s += """</body>"""
request.write(s)
request.write(to_bytes(s))
request.finish()
class Delay(LeafResource):
def render_GET(self, request):
n = getarg(request, "n", 1, type=float)
b = getarg(request, "b", 1, type=int)
n = getarg(request, b"n", 1, type=float)
b = getarg(request, b"b", 1, type=int)
if b:
# send headers now and delay body
request.write('')
@ -93,16 +96,16 @@ class Delay(LeafResource):
return NOT_DONE_YET
def _delayedRender(self, request, n):
request.write("Response delayed for %0.3f seconds\n" % n)
request.write(to_bytes("Response delayed for %0.3f seconds\n" % n))
request.finish()
class Status(LeafResource):
def render_GET(self, request):
n = getarg(request, "n", 200, type=int)
n = getarg(request, b"n", 200, type=int)
request.setResponseCode(n)
return ""
return b""
class Raw(LeafResource):
@ -114,7 +117,7 @@ class Raw(LeafResource):
render_POST = render_GET
def _delayedRender(self, request):
raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n')
raw = getarg(request, b'raw', b'HTTP 1.1 200 OK\n')
request.startedWriting = 1
request.write(raw)
request.channel.transport.loseConnection()
@ -125,10 +128,12 @@ class Echo(LeafResource):
def render_GET(self, request):
output = {
'headers': dict(request.requestHeaders.getAllRawHeaders()),
'body': request.content.read(),
'headers': dict(
(to_unicode(k), [to_unicode(v) for v in vs])
for k, vs in request.requestHeaders.getAllRawHeaders()),
'body': to_unicode(request.content.read()),
}
return json.dumps(output)
return to_bytes(json.dumps(output))
class Partial(LeafResource):
@ -146,7 +151,7 @@ class Partial(LeafResource):
class Drop(Partial):
def _delayedRender(self, request):
abort = getarg(request, "abort", 0, type=int)
abort = getarg(request, b"abort", 0, type=int)
request.write(b"this connection will be dropped\n")
tr = request.channel.transport
try:

2
tests/py3-ignores.txt
View File

@ -1,7 +1,5 @@
tests/test_closespider.py
tests/test_exporters.py
tests/test_linkextractors_deprecated.py
tests/test_crawl.py
tests/test_mail.py
tests/test_pipeline_files.py
tests/test_pipeline_images.py

2
tests/spiders.py
View File

@ -119,7 +119,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
if self.fail_before_yield:
1 / 0
for s in xrange(100):
for s in range(100):
qargs = {'total': 10, 'seed': s}
url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1)
yield Request(url, meta={'seed': s})

9
tests/test_crawl.py
View File

@ -8,6 +8,7 @@ from twisted.trial.unittest import TestCase
from scrapy.http import Request
from scrapy.crawler import CrawlerRunner
from scrapy.utils.python import to_unicode
from tests import mock
from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
@ -201,16 +202,16 @@ with multiples lines
self.assertIn('responses', crawler.spider.meta)
self.assertNotIn('failures', crawler.spider.meta)
# start requests doesn't set Referer header
echo0 = json.loads(crawler.spider.meta['responses'][2].body)
echo0 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body))
self.assertNotIn('Referer', echo0['headers'])
# following request sets Referer to start request url
echo1 = json.loads(crawler.spider.meta['responses'][1].body)
echo1 = json.loads(to_unicode(crawler.spider.meta['responses'][1].body))
self.assertEqual(echo1['headers'].get('Referer'), [req0.url])
# next request avoids Referer header
echo2 = json.loads(crawler.spider.meta['responses'][2].body)
echo2 = json.loads(to_unicode(crawler.spider.meta['responses'][2].body))
self.assertNotIn('Referer', echo2['headers'])
# last request explicitly sets a Referer header
echo3 = json.loads(crawler.spider.meta['responses'][3].body)
echo3 = json.loads(to_unicode(crawler.spider.meta['responses'][3].body))
self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com'])
@defer.inlineCallbacks