RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1470 from scrapy/py3-middlewares 

[MRG] port some downloader middlewares to Python 3
This commit is contained in:
Daniel Graña 2015-09-02 00:17:37 -03:00
parent bc499cb552 c44cafe4f5
commit 3919ad64c5
8 changed files with 43 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/downloadermiddlewares/httpauth.py
View File

@ -27,5 +27,5 @@ class HttpAuthMiddleware(object):
def process_request(self, request, spider):
auth = getattr(self, 'auth', None)
if auth and 'Authorization' not in request.headers:
request.headers['Authorization'] = auth
if auth and b'Authorization' not in request.headers:
request.headers[b'Authorization'] = auth

2
scrapy/downloadermiddlewares/useragent.py
View File

@ -20,4 +20,4 @@ class UserAgentMiddleware(object):
def process_request(self, request, spider):
if self.user_agent:
request.headers.setdefault('User-Agent', self.user_agent)
request.headers.setdefault(b'User-Agent', self.user_agent)

6
tests/py3-ignores.txt
View File

@ -6,18 +6,12 @@ tests/test_exporters.py
tests/test_linkextractors_deprecated.py
tests/test_crawl.py
tests/test_downloader_handlers.py
tests/test_downloadermiddleware_ajaxcrawlable.py
tests/test_downloadermiddleware_defaultheaders.py
tests/test_downloadermiddleware_downloadtimeout.py
tests/test_downloadermiddleware_httpauth.py
tests/test_downloadermiddleware_httpcache.py
tests/test_downloadermiddleware_httpcompression.py
tests/test_downloadermiddleware_httpproxy.py
tests/test_downloadermiddleware.py
tests/test_downloadermiddleware_redirect.py
tests/test_downloadermiddleware_retry.py
tests/test_downloadermiddleware_stats.py
tests/test_downloadermiddleware_useragent.py
tests/test_engine.py
tests/test_mail.py
tests/test_pipeline_files.py

4
tests/test_downloadermiddleware_ajaxcrawlable.py
View File

@ -14,7 +14,7 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase):
self.mw = AjaxCrawlMiddleware.from_crawler(crawler)
def _ajaxcrawlable_body(self):
return '<html><head><meta name="fragment" content="!"/></head><body></body></html>'
return b'<html><head><meta name="fragment" content="!"/></head><body></body></html>'
def _req_resp(self, url, req_kwargs=None, resp_kwargs=None):
req = Request(url, **(req_kwargs or {}))
@ -53,6 +53,6 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase):
assert resp3 is resp2
def test_noncrawlable_body(self):
req, resp = self._req_resp('http://example.com/', {}, {'body': '<html></html>'})
req, resp = self._req_resp('http://example.com/', {}, {'body': b'<html></html>'})
resp2 = self.mw.process_response(req, resp, self.spider)
self.assertIs(resp, resp2)

13
tests/test_downloadermiddleware_defaultheaders.py
View File

@ -1,10 +1,10 @@
from unittest import TestCase
import six
from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from scrapy.utils.python import to_bytes
class TestDefaultHeadersMiddleware(TestCase):
@ -12,8 +12,10 @@ class TestDefaultHeadersMiddleware(TestCase):
def get_defaults_spider_mw(self):
crawler = get_crawler(Spider)
spider = crawler._create_spider('foo')
defaults = dict([(k, [v]) for k, v in \
six.iteritems(crawler.settings.get('DEFAULT_REQUEST_HEADERS'))])
defaults = {
to_bytes(k): [to_bytes(v)]
for k, v in crawler.settings.get('DEFAULT_REQUEST_HEADERS').items()
}
return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler)
def test_process_request(self):
@ -25,9 +27,10 @@ class TestDefaultHeadersMiddleware(TestCase):
def test_update_headers(self):
defaults, spider, mw = self.get_defaults_spider_mw()
headers = {'Accept-Language': ['es'], 'Test-Header': ['test']}
bytes_headers = {b'Accept-Language': [b'es'], b'Test-Header': [b'test']}
req = Request('http://www.scrapytest.org', headers=headers)
self.assertEquals(req.headers, headers)
self.assertEquals(req.headers, bytes_headers)
mw.process_request(req, spider)
defaults.update(headers)
defaults.update(bytes_headers)
self.assertEquals(req.headers, defaults)

13
tests/test_downloadermiddleware_httpauth.py
View File

@ -4,10 +4,12 @@ from scrapy.http import Request
from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
from scrapy.spiders import Spider
class TestSpider(Spider):
http_user = 'foo'
http_pass = 'bar'
class HttpAuthMiddlewareTest(unittest.TestCase):
def setUp(self):
@ -21,13 +23,10 @@ class HttpAuthMiddlewareTest(unittest.TestCase):
def test_auth(self):
req = Request('http://scrapytest.org/')
assert self.mw.process_request(req, self.spider) is None
self.assertEquals(req.headers['Authorization'], 'Basic Zm9vOmJhcg==')
self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==')
def test_auth_already_set(self):
req = Request('http://scrapytest.org/', headers=dict(Authorization='Digest 123'))
req = Request('http://scrapytest.org/',
headers=dict(Authorization='Digest 123'))
assert self.mw.process_request(req, self.spider) is None
self.assertEquals(req.headers['Authorization'], 'Digest 123')
if __name__ == '__main__':
unittest.main()
self.assertEquals(req.headers['Authorization'], b'Digest 123')

27
tests/test_downloadermiddleware_stats.py
View File

@ -6,6 +6,10 @@ from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
class MyException(Exception):
pass
class TestDownloaderStats(TestCase):
def setUp(self):
@ -18,21 +22,28 @@ class TestDownloaderStats(TestCase):
self.req = Request('http://scrapytest.org')
self.res = Response('scrapytest.org', status=400)
def assertStatsEqual(self, key, value):
self.assertEqual(
self.crawler.stats.get_value(key, spider=self.spider),
value,
str(self.crawler.stats.get_stats(self.spider))
)
def test_process_request(self):
self.mw.process_request(self.req, self.spider)
self.assertEqual(self.crawler.stats.get_value('downloader/request_count', \
spider=self.spider), 1)
self.assertStatsEqual('downloader/request_count', 1)
def test_process_response(self):
self.mw.process_response(self.req, self.res, self.spider)
self.assertEqual(self.crawler.stats.get_value('downloader/response_count', \
spider=self.spider), 1)
self.assertStatsEqual('downloader/response_count', 1)
def test_process_exception(self):
self.mw.process_exception(self.req, Exception(), self.spider)
self.assertEqual(self.crawler.stats.get_value('downloader/exception_count', \
spider=self.spider), 1)
self.mw.process_exception(self.req, MyException(), self.spider)
self.assertStatsEqual('downloader/exception_count', 1)
self.assertStatsEqual(
'downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException',
1
)
def tearDown(self):
self.crawler.stats.close_spider(self.spider, '')

9
tests/test_downloadermiddleware_useragent.py
View File

@ -17,7 +17,7 @@ class UserAgentMiddlewareTest(TestCase):
spider, mw = self.get_spider_and_mw('default_useragent')
req = Request('http://scrapytest.org/')
assert mw.process_request(req, spider) is None
self.assertEquals(req.headers['User-Agent'], 'default_useragent')
self.assertEquals(req.headers['User-Agent'], b'default_useragent')
def test_remove_agent(self):
# settings UESR_AGENT to None should remove the user agent
@ -34,15 +34,16 @@ class UserAgentMiddlewareTest(TestCase):
mw.spider_opened(spider)
req = Request('http://scrapytest.org/')
assert mw.process_request(req, spider) is None
self.assertEquals(req.headers['User-Agent'], 'spider_useragent')
self.assertEquals(req.headers['User-Agent'], b'spider_useragent')
def test_header_agent(self):
spider, mw = self.get_spider_and_mw('default_useragent')
spider.user_agent = 'spider_useragent'
mw.spider_opened(spider)
req = Request('http://scrapytest.org/', headers={'User-Agent': 'header_useragent'})
req = Request('http://scrapytest.org/',
headers={'User-Agent': 'header_useragent'})
assert mw.process_request(req, spider) is None
self.assertEquals(req.headers['User-Agent'], 'header_useragent')
self.assertEquals(req.headers['User-Agent'], b'header_useragent')
def test_no_agent(self):
spider, mw = self.get_spider_and_mw(None)