diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 7854e899a..7aa7a62bc 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -27,5 +27,5 @@ class HttpAuthMiddleware(object): def process_request(self, request, spider): auth = getattr(self, 'auth', None) - if auth and 'Authorization' not in request.headers: - request.headers['Authorization'] = auth + if auth and b'Authorization' not in request.headers: + request.headers[b'Authorization'] = auth diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index 17793cad7..d24750c69 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -20,4 +20,4 @@ class UserAgentMiddleware(object): def process_request(self, request, spider): if self.user_agent: - request.headers.setdefault('User-Agent', self.user_agent) + request.headers.setdefault(b'User-Agent', self.user_agent) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 759eeffff..95e2181e3 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -6,18 +6,12 @@ tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py tests/test_downloader_handlers.py -tests/test_downloadermiddleware_ajaxcrawlable.py -tests/test_downloadermiddleware_defaultheaders.py -tests/test_downloadermiddleware_downloadtimeout.py -tests/test_downloadermiddleware_httpauth.py tests/test_downloadermiddleware_httpcache.py tests/test_downloadermiddleware_httpcompression.py tests/test_downloadermiddleware_httpproxy.py tests/test_downloadermiddleware.py tests/test_downloadermiddleware_redirect.py tests/test_downloadermiddleware_retry.py -tests/test_downloadermiddleware_stats.py -tests/test_downloadermiddleware_useragent.py tests/test_engine.py tests/test_mail.py tests/test_pipeline_files.py diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 67c57778d..493691ea4 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -14,7 +14,7 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): self.mw = AjaxCrawlMiddleware.from_crawler(crawler) def _ajaxcrawlable_body(self): - return '' + return b'' def _req_resp(self, url, req_kwargs=None, resp_kwargs=None): req = Request(url, **(req_kwargs or {})) @@ -53,6 +53,6 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): assert resp3 is resp2 def test_noncrawlable_body(self): - req, resp = self._req_resp('http://example.com/', {}, {'body': ''}) + req, resp = self._req_resp('http://example.com/', {}, {'body': b''}) resp2 = self.mw.process_response(req, resp, self.spider) self.assertIs(resp, resp2) diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 75d8a1921..80efa83f9 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -1,10 +1,10 @@ from unittest import TestCase -import six from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from scrapy.utils.python import to_bytes class TestDefaultHeadersMiddleware(TestCase): @@ -12,8 +12,10 @@ class TestDefaultHeadersMiddleware(TestCase): def get_defaults_spider_mw(self): crawler = get_crawler(Spider) spider = crawler._create_spider('foo') - defaults = dict([(k, [v]) for k, v in \ - six.iteritems(crawler.settings.get('DEFAULT_REQUEST_HEADERS'))]) + defaults = { + to_bytes(k): [to_bytes(v)] + for k, v in crawler.settings.get('DEFAULT_REQUEST_HEADERS').items() + } return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler) def test_process_request(self): @@ -25,9 +27,10 @@ class TestDefaultHeadersMiddleware(TestCase): def test_update_headers(self): defaults, spider, mw = self.get_defaults_spider_mw() headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} + bytes_headers = {b'Accept-Language': [b'es'], b'Test-Header': [b'test']} req = Request('http://www.scrapytest.org', headers=headers) - self.assertEquals(req.headers, headers) + self.assertEquals(req.headers, bytes_headers) mw.process_request(req, spider) - defaults.update(headers) + defaults.update(bytes_headers) self.assertEquals(req.headers, defaults) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index c30fa97c6..425a5cc79 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -4,10 +4,12 @@ from scrapy.http import Request from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spiders import Spider + class TestSpider(Spider): http_user = 'foo' http_pass = 'bar' + class HttpAuthMiddlewareTest(unittest.TestCase): def setUp(self): @@ -21,13 +23,10 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def test_auth(self): req = Request('http://scrapytest.org/') assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], 'Basic Zm9vOmJhcg==') + self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') def test_auth_already_set(self): - req = Request('http://scrapytest.org/', headers=dict(Authorization='Digest 123')) + req = Request('http://scrapytest.org/', + headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], 'Digest 123') - - -if __name__ == '__main__': - unittest.main() + self.assertEquals(req.headers['Authorization'], b'Digest 123') diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index fb46ccff6..1f2616e35 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -6,6 +6,10 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +class MyException(Exception): + pass + + class TestDownloaderStats(TestCase): def setUp(self): @@ -18,21 +22,28 @@ class TestDownloaderStats(TestCase): self.req = Request('http://scrapytest.org') self.res = Response('scrapytest.org', status=400) + def assertStatsEqual(self, key, value): + self.assertEqual( + self.crawler.stats.get_value(key, spider=self.spider), + value, + str(self.crawler.stats.get_stats(self.spider)) + ) + def test_process_request(self): self.mw.process_request(self.req, self.spider) - self.assertEqual(self.crawler.stats.get_value('downloader/request_count', \ - spider=self.spider), 1) + self.assertStatsEqual('downloader/request_count', 1) def test_process_response(self): self.mw.process_response(self.req, self.res, self.spider) - self.assertEqual(self.crawler.stats.get_value('downloader/response_count', \ - spider=self.spider), 1) + self.assertStatsEqual('downloader/response_count', 1) def test_process_exception(self): - self.mw.process_exception(self.req, Exception(), self.spider) - self.assertEqual(self.crawler.stats.get_value('downloader/exception_count', \ - spider=self.spider), 1) + self.mw.process_exception(self.req, MyException(), self.spider) + self.assertStatsEqual('downloader/exception_count', 1) + self.assertStatsEqual( + 'downloader/exception_type_count/tests.test_downloadermiddleware_stats.MyException', + 1 + ) def tearDown(self): self.crawler.stats.close_spider(self.spider, '') - diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 741c8de76..1e41fdace 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -17,7 +17,7 @@ class UserAgentMiddlewareTest(TestCase): spider, mw = self.get_spider_and_mw('default_useragent') req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], 'default_useragent') + self.assertEquals(req.headers['User-Agent'], b'default_useragent') def test_remove_agent(self): # settings UESR_AGENT to None should remove the user agent @@ -34,15 +34,16 @@ class UserAgentMiddlewareTest(TestCase): mw.spider_opened(spider) req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], 'spider_useragent') + self.assertEquals(req.headers['User-Agent'], b'spider_useragent') def test_header_agent(self): spider, mw = self.get_spider_and_mw('default_useragent') spider.user_agent = 'spider_useragent' mw.spider_opened(spider) - req = Request('http://scrapytest.org/', headers={'User-Agent': 'header_useragent'}) + req = Request('http://scrapytest.org/', + headers={'User-Agent': 'header_useragent'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], 'header_useragent') + self.assertEquals(req.headers['User-Agent'], b'header_useragent') def test_no_agent(self): spider, mw = self.get_spider_and_mw(None)