From c6adf648dcfe59b52c0e4663e701a232e78d7bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 3 Aug 2015 16:28:29 -0300 Subject: [PATCH] PY3 port COOKIES_DEBUG and add tests --- scrapy/downloadermiddlewares/cookies.py | 15 +++-- scrapy/mail.py | 3 +- tests/test_downloadermiddleware_cookies.py | 64 +++++++++++++++++++++- 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 270d621cd..321c0171b 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -6,6 +6,7 @@ from collections import defaultdict from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar +from scrapy.utils.python import to_native_str logger = logging.getLogger(__name__) @@ -52,18 +53,20 @@ class CookiesMiddleware(object): def _debug_cookie(self, request, spider): if self.debug: - cl = request.headers.getlist('Cookie') + cl = [to_native_str(c, errors='replace') + for c in request.headers.getlist('Cookie')] if cl: - msg = "Sending cookies to: %s" % request + os.linesep - msg += os.linesep.join("Cookie: %s" % c for c in cl) + cookies = "\n".join("Cookie: {}\n".format(c) for c in cl) + msg = "Sending cookies to: {}\n{}".format(request, cookies) logger.debug(msg, extra={'spider': spider}) def _debug_set_cookie(self, response, spider): if self.debug: - cl = response.headers.getlist('Set-Cookie') + cl = [to_native_str(c, errors='replace') + for c in response.headers.getlist('Set-Cookie')] if cl: - msg = "Received cookies from: %s" % response + os.linesep - msg += os.linesep.join("Set-Cookie: %s" % c for c in cl) + cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl) + msg = "Received cookies from: {}\n{}".format(response, cookies) logger.debug(msg, extra={'spider': spider}) def _format_cookie(self, cookie): diff --git a/scrapy/mail.py b/scrapy/mail.py index 2b4c57980..ad8ecbe13 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -20,7 +20,6 @@ else: from email import encoders as Encoders from twisted.internet import defer, reactor, ssl -from twisted.mail.smtp import ESMTPSenderFactory logger = logging.getLogger(__name__) @@ -102,6 +101,8 @@ class MailSender(object): 'mailattachs': nattachs, 'mailerr': errstr}) def _sendmail(self, to_addrs, msg): + # Import twisted.mail here because it is not available in python3 + from twisted.mail.smtp import ESMTPSenderFactory msg = StringIO(msg) d = defer.Deferred() factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \ diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 63be0beb8..66d9faa79 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,8 +1,12 @@ -from unittest import TestCase import re +import logging +from unittest import TestCase +from testfixtures import LogCapture from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from scrapy.exceptions import NotConfigured from scrapy.downloadermiddlewares.cookies import CookiesMiddleware @@ -34,6 +38,64 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req2, self.spider) is None self.assertEquals(req2.headers.get('Cookie'), b"C1=value1") + def test_setting_false_cookies_enabled(self): + self.assertRaises( + NotConfigured, + CookiesMiddleware.from_crawler, + get_crawler(settings_dict={'COOKIES_ENABLED': False}) + ) + + def test_setting_default_cookies_enabled(self): + self.assertIsInstance( + CookiesMiddleware.from_crawler(get_crawler()), + CookiesMiddleware + ) + + def test_setting_true_cookies_enabled(self): + self.assertIsInstance( + CookiesMiddleware.from_crawler( + get_crawler(settings_dict={'COOKIES_ENABLED': True}) + ), + CookiesMiddleware + ) + + def test_setting_enabled_cookies_debug(self): + crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True}) + mw = CookiesMiddleware.from_crawler(crawler) + with LogCapture('scrapy.downloadermiddlewares.cookies', + level=logging.DEBUG) as l: + req = Request('http://scrapytest.org/') + res = Response('http://scrapytest.org/', + headers={'Set-Cookie': 'C1=value1; path=/'}) + mw.process_response(req, res, crawler.spider) + req2 = Request('http://scrapytest.org/sub1/') + mw.process_request(req2, crawler.spider) + + l.check( + ('scrapy.downloadermiddlewares.cookies', + 'DEBUG', + 'Received cookies from: <200 http://scrapytest.org/>\n' + 'Set-Cookie: C1=value1; path=/\n'), + ('scrapy.downloadermiddlewares.cookies', + 'DEBUG', + 'Sending cookies to: \n' + 'Cookie: C1=value1\n'), + ) + + def test_setting_disabled_cookies_debug(self): + crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False}) + mw = CookiesMiddleware.from_crawler(crawler) + with LogCapture('scrapy.downloadermiddlewares.cookies', + level=logging.DEBUG) as l: + req = Request('http://scrapytest.org/') + res = Response('http://scrapytest.org/', + headers={'Set-Cookie': 'C1=value1; path=/'}) + mw.process_response(req, res, crawler.spider) + req2 = Request('http://scrapytest.org/sub1/') + mw.process_request(req2, crawler.spider) + + l.check() + def test_do_not_break_on_non_utf8_header(self): req = Request('http://scrapytest.org/') assert self.mw.process_request(req, self.spider) is None